{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5816066884769174, "eval_steps": 200.0, "global_step": 800, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 223.25, "completions/mean_length": 184.6875, "completions/min_length": 154.75, "epoch": 0.0007270083605961468, "frac_reward_zero_std": 0.0, "grad_norm": 1.2505969216612862, "kl": 0.0, "learning_rate": 1.4492753623188406e-08, "loss": -8.940696716308594e-08, "reward": 1.800000011920929, "reward_std": 0.16622530855238438, "rewards/MveiAccuracyReward/mean": 0.96875, "rewards/MveiAccuracyReward/std": 0.0883883461356163, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.33124999701976776, "rewards/MveiLLMConsistencyReward/std": 0.11712000705301762, "step": 1, "step_time": 88.10392547957599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 204.9375, "completions/mean_length": 179.125, "completions/min_length": 158.4375, "epoch": 0.0036350418029807343, "frac_reward_zero_std": 0.0, "grad_norm": 1.324666224992261, "kl": 7.218225368887943e-05, "learning_rate": 7.246376811594203e-08, "loss": 2.7120113372802734e-06, "reward": 1.9343749806284904, "reward_std": 0.22219385765492916, "rewards/MveiAccuracyReward/mean": 0.96875, "rewards/MveiAccuracyReward/std": 0.07312605157494545, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.46562500670552254, "rewards/MveiLLMConsistencyReward/std": 0.18109685834497213, "step": 5, "step_time": 80.04274107748643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 203.9, "completions/mean_length": 177.2, "completions/min_length": 154.55, "epoch": 0.007270083605961469, "frac_reward_zero_std": 0.1, "grad_norm": 1.3475511420920507, "kl": 9.369138433612534e-05, "learning_rate": 1.4492753623188405e-07, "loss": 3.824383020401001e-06, "reward": 1.7812500029802323, "reward_std": 0.17669749669730664, "rewards/MveiAccuracyReward/mean": 0.85, "rewards/MveiAccuracyReward/std": 0.07891046851873398, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.4312500014901161, "rewards/MveiLLMConsistencyReward/std": 0.1377484068274498, "step": 10, "step_time": 81.79588728807866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 218.5, "completions/mean_length": 188.91875, "completions/min_length": 163.95, "epoch": 0.010905125408942203, "frac_reward_zero_std": 0.1, "grad_norm": 1.162502802224763, "kl": 8.805416291579604e-05, "learning_rate": 2.1739130434782607e-07, "loss": 3.489851951599121e-06, "reward": 1.7925000011920929, "reward_std": 0.2501345627009869, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.1473084270954132, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.380000002682209, "rewards/MveiLLMConsistencyReward/std": 0.14857573956251144, "step": 15, "step_time": 78.87650614492595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 217.85, "completions/mean_length": 184.3125, "completions/min_length": 159.75, "epoch": 0.014540167211922937, "frac_reward_zero_std": 0.2, "grad_norm": 1.3559448301270443, "kl": 9.812471289478708e-05, "learning_rate": 2.898550724637681e-07, "loss": 3.826618194580078e-06, "reward": 1.8037499845027924, "reward_std": 0.1779584601521492, "rewards/MveiAccuracyReward/mean": 0.9625, "rewards/MveiAccuracyReward/std": 0.09385617971420288, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.34125000759959223, "rewards/MveiLLMConsistencyReward/std": 0.10352658182382583, "step": 20, "step_time": 78.4549627777189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 222.8, "completions/mean_length": 179.50625, "completions/min_length": 153.45, "epoch": 0.01817520901490367, "frac_reward_zero_std": 0.15, "grad_norm": 1.073699055517611, "kl": 0.00010079531693918398, "learning_rate": 3.6231884057971015e-07, "loss": 4.154443740844727e-06, "reward": 1.8262500017881393, "reward_std": 0.2465540524572134, "rewards/MveiAccuracyReward/mean": 0.85, "rewards/MveiAccuracyReward/std": 0.11845555454492569, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.4762500025331974, "rewards/MveiLLMConsistencyReward/std": 0.173562653362751, "step": 25, "step_time": 83.93649914860725 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 206.25, "completions/mean_length": 178.7, "completions/min_length": 154.7, "epoch": 0.021810250817884406, "frac_reward_zero_std": 0.1, "grad_norm": 1.2915238732276229, "kl": 9.987151315726805e-05, "learning_rate": 4.3478260869565214e-07, "loss": 3.7729740142822264e-06, "reward": 1.7924999475479126, "reward_std": 0.15293546877801417, "rewards/MveiAccuracyReward/mean": 0.88125, "rewards/MveiAccuracyReward/std": 0.025877460837364197, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.41125001236796377, "rewards/MveiLLMConsistencyReward/std": 0.14563345164060593, "step": 30, "step_time": 74.32066762559116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 209.6, "completions/mean_length": 181.1, "completions/min_length": 159.25, "epoch": 0.02544529262086514, "frac_reward_zero_std": 0.05, "grad_norm": 1.2323526862059118, "kl": 9.222824437529197e-05, "learning_rate": 5.072463768115942e-07, "loss": 3.7275254726409913e-06, "reward": 2.002499985694885, "reward_std": 0.16539971344172955, "rewards/MveiAccuracyReward/mean": 0.98125, "rewards/MveiAccuracyReward/std": 0.05303300768136978, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.5212500058114529, "rewards/MveiLLMConsistencyReward/std": 0.14334528222680093, "step": 35, "step_time": 81.8711244918406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 207.85, "completions/mean_length": 179.81875, "completions/min_length": 159.1, "epoch": 0.029080334423845874, "frac_reward_zero_std": 0.05, "grad_norm": 1.3340606496373275, "kl": 9.716968615975929e-05, "learning_rate": 5.797101449275362e-07, "loss": 3.904104232788086e-06, "reward": 1.9056250035762787, "reward_std": 0.19380234889686107, "rewards/MveiAccuracyReward/mean": 0.9375, "rewards/MveiAccuracyReward/std": 0.0816463440656662, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.47124999910593035, "rewards/MveiLLMConsistencyReward/std": 0.1763294253498316, "step": 40, "step_time": 81.85793585814535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 209.55, "completions/mean_length": 184.9875, "completions/min_length": 160.55, "epoch": 0.03271537622682661, "frac_reward_zero_std": 0.25, "grad_norm": 1.2263462985241742, "kl": 0.00014888912246533437, "learning_rate": 6.521739130434782e-07, "loss": 5.656480789184571e-06, "reward": 1.9612499594688415, "reward_std": 0.1264342725276947, "rewards/MveiAccuracyReward/mean": 0.99375, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.467500015348196, "rewards/MveiLLMConsistencyReward/std": 0.12135031558573246, "step": 45, "step_time": 79.34563678354024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 216.65, "completions/mean_length": 184.98125, "completions/min_length": 161.9, "epoch": 0.03635041802980734, "frac_reward_zero_std": 0.05, "grad_norm": 1.3899616519162392, "kl": 0.00016749764145060909, "learning_rate": 7.246376811594203e-07, "loss": 6.7442655563354496e-06, "reward": 1.9762499809265137, "reward_std": 0.18204652182757855, "rewards/MveiAccuracyReward/mean": 0.96875, "rewards/MveiAccuracyReward/std": 0.06123279929161072, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.5075000047683715, "rewards/MveiLLMConsistencyReward/std": 0.17018421925604343, "step": 50, "step_time": 90.00479119606317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 202.05, "completions/mean_length": 177.2625, "completions/min_length": 154.5, "epoch": 0.039985459832788076, "frac_reward_zero_std": 0.05, "grad_norm": 1.3463806058504582, "kl": 0.00022680436522932724, "learning_rate": 7.971014492753623e-07, "loss": 8.958578109741212e-06, "reward": 1.718749988079071, "reward_std": 0.2583191357553005, "rewards/MveiAccuracyReward/mean": 0.76875, "rewards/MveiAccuracyReward/std": 0.11973364055156707, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.45000000223517417, "rewards/MveiLLMConsistencyReward/std": 0.17364961095154285, "step": 55, "step_time": 83.08356610722839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 204.4, "completions/mean_length": 180.9375, "completions/min_length": 158.5, "epoch": 0.04362050163576881, "frac_reward_zero_std": 0.15, "grad_norm": 1.0034107978808375, "kl": 0.0002776149965939112, "learning_rate": 8.695652173913043e-07, "loss": 1.1054426431655884e-05, "reward": 1.869999971985817, "reward_std": 0.21551885977387428, "rewards/MveiAccuracyReward/mean": 0.8375, "rewards/MveiAccuracyReward/std": 0.08984613567590713, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.5325000062584877, "rewards/MveiLLMConsistencyReward/std": 0.1620310701429844, "step": 60, "step_time": 75.35761900916695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 201.9, "completions/mean_length": 177.61875, "completions/min_length": 157.7, "epoch": 0.04725554343874955, "frac_reward_zero_std": 0.05, "grad_norm": 1.2967341052700547, "kl": 0.0006032844808942173, "learning_rate": 9.420289855072463e-07, "loss": 2.4043023586273193e-05, "reward": 1.8174999833106995, "reward_std": 0.22795652821660042, "rewards/MveiAccuracyReward/mean": 0.88125, "rewards/MveiAccuracyReward/std": 0.1075238049030304, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.4362500086426735, "rewards/MveiLLMConsistencyReward/std": 0.16589595302939414, "step": 65, "step_time": 69.4659463264048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 227.7, "completions/mean_length": 187.575, "completions/min_length": 162.35, "epoch": 0.05089058524173028, "frac_reward_zero_std": 0.15, "grad_norm": 1.2025105509627323, "kl": 0.0006610367418034002, "learning_rate": 9.999985555970398e-07, "loss": 2.6616454124450684e-05, "reward": 1.693750038743019, "reward_std": 0.21350354701280594, "rewards/MveiAccuracyReward/mean": 0.7625, "rewards/MveiAccuracyReward/std": 0.19401475489139558, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.4312499947845936, "rewards/MveiLLMConsistencyReward/std": 0.18215158805251122, "step": 70, "step_time": 82.89062785804272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 210.3, "completions/mean_length": 183.86875, "completions/min_length": 160.4, "epoch": 0.05452562704471101, "frac_reward_zero_std": 0.05, "grad_norm": 1.274740408207357, "kl": 0.000759283323714044, "learning_rate": 9.999480023696746e-07, "loss": 3.0159205198287965e-05, "reward": 1.7981249630451202, "reward_std": 0.20472086891531943, "rewards/MveiAccuracyReward/mean": 0.90625, "rewards/MveiAccuracyReward/std": 0.09932401329278946, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.395000009983778, "rewards/MveiLLMConsistencyReward/std": 0.13778294138610364, "step": 75, "step_time": 84.61769868656992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 216.5, "completions/mean_length": 185.40625, "completions/min_length": 160.45, "epoch": 0.05816066884769175, "frac_reward_zero_std": 0.05, "grad_norm": 1.2841827611979246, "kl": 0.0011498910709633492, "learning_rate": 9.998252373392842e-07, "loss": 4.5952200889587405e-05, "reward": 1.974999985098839, "reward_std": 0.18783026114106177, "rewards/MveiAccuracyReward/mean": 0.9375, "rewards/MveiAccuracyReward/std": 0.05850084125995636, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.5375000052154064, "rewards/MveiLLMConsistencyReward/std": 0.15051332265138626, "step": 80, "step_time": 83.98414503261446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 229.2, "completions/mean_length": 181.86875, "completions/min_length": 156.8, "epoch": 0.061795710650672485, "frac_reward_zero_std": 0.15, "grad_norm": 0.9487729565385968, "kl": 0.0016302011121297256, "learning_rate": 9.996302782378808e-07, "loss": 6.515383720397949e-05, "reward": 1.924374994635582, "reward_std": 0.18220550268888475, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.06208146214485168, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.5150000043213367, "rewards/MveiLLMConsistencyReward/std": 0.14921745620667934, "step": 85, "step_time": 84.90599675662816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 219.6, "completions/mean_length": 185.33125, "completions/min_length": 157.2, "epoch": 0.06543075245365322, "frac_reward_zero_std": 0.1, "grad_norm": 1.2833217364089817, "kl": 0.0021225117961876094, "learning_rate": 9.993631532250894e-07, "loss": 8.470714092254639e-05, "reward": 2.0143749833106996, "reward_std": 0.20510876551270485, "rewards/MveiAccuracyReward/mean": 0.975, "rewards/MveiAccuracyReward/std": 0.026726123690605164, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.5425000056624413, "rewards/MveiLLMConsistencyReward/std": 0.1708848036825657, "step": 90, "step_time": 88.06912175342441 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 216.95, "completions/mean_length": 184.525, "completions/min_length": 161.65, "epoch": 0.06906579425663395, "frac_reward_zero_std": 0.25, "grad_norm": 1.3513061878234627, "kl": 0.002421390629024245, "learning_rate": 9.990239008840802e-07, "loss": 9.694695472717285e-05, "reward": 1.7700000196695327, "reward_std": 0.20398809798061848, "rewards/MveiAccuracyReward/mean": 0.85, "rewards/MveiAccuracyReward/std": 0.046291005611419675, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.41999999806284904, "rewards/MveiLLMConsistencyReward/std": 0.15986914485692977, "step": 95, "step_time": 82.67934676595033 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 212.55, "completions/mean_length": 187.35625, "completions/min_length": 162.25, "epoch": 0.07270083605961468, "frac_reward_zero_std": 0.2, "grad_norm": 1.0309971338238968, "kl": 0.004161263263085857, "learning_rate": 9.986125702159954e-07, "loss": 0.00016637742519378662, "reward": 1.9337499886751175, "reward_std": 0.19087109677493572, "rewards/MveiAccuracyReward/mean": 0.93125, "rewards/MveiAccuracyReward/std": 0.06123279929161072, "rewards/MveiFormatReward/mean": 0.9875, "rewards/MveiFormatReward/std": 0.03535533845424652, "rewards/MveiLLMConsistencyReward/mean": 0.5087500065565109, "rewards/MveiLLMConsistencyReward/std": 0.16509965918958186, "step": 100, "step_time": 80.46456340812146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 217.95, "completions/mean_length": 188.29375, "completions/min_length": 161.45, "epoch": 0.07633587786259542, "frac_reward_zero_std": 0.1, "grad_norm": 1.1190120423614052, "kl": 0.0048583348863758145, "learning_rate": 9.98129220632872e-07, "loss": 0.00019429922103881837, "reward": 1.9524999916553498, "reward_std": 0.234458002820611, "rewards/MveiAccuracyReward/mean": 0.8875, "rewards/MveiAccuracyReward/std": 0.08711026012897491, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.5650000050663948, "rewards/MveiLLMConsistencyReward/std": 0.18389294520020485, "step": 105, "step_time": 78.62311269156635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 228.0, "completions/mean_length": 195.41875, "completions/min_length": 166.15, "epoch": 0.07997091966557615, "frac_reward_zero_std": 0.15, "grad_norm": 1.2835284316608364, "kl": 0.005961388075957075, "learning_rate": 9.975739219490601e-07, "loss": 0.00023860633373260499, "reward": 1.969374990463257, "reward_std": 0.20738761276006698, "rewards/MveiAccuracyReward/mean": 0.925, "rewards/MveiAccuracyReward/std": 0.0816463440656662, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.5475000068545341, "rewards/MveiLLMConsistencyReward/std": 0.17429551668465137, "step": 110, "step_time": 78.5548978485167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.75, "completions/mean_length": 193.85625, "completions/min_length": 165.65, "epoch": 0.0836059614685569, "frac_reward_zero_std": 0.1, "grad_norm": 1.1174049789751839, "kl": 0.007999175693839788, "learning_rate": 9.969467543711395e-07, "loss": 0.00031993985176086425, "reward": 1.8487499833106995, "reward_std": 0.31319191604852675, "rewards/MveiAccuracyReward/mean": 0.8625, "rewards/MveiAccuracyReward/std": 0.17233722507953644, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.4862500101327896, "rewards/MveiLLMConsistencyReward/std": 0.2090558473020792, "step": 115, "step_time": 81.70319018065929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 225.25, "completions/mean_length": 189.41875, "completions/min_length": 162.3, "epoch": 0.08724100327153762, "frac_reward_zero_std": 0.05, "grad_norm": 1.4027297985697493, "kl": 0.009565949090756476, "learning_rate": 9.962478084863336e-07, "loss": 0.00038267150521278384, "reward": 1.8074999898672104, "reward_std": 0.2678529404103756, "rewards/MveiAccuracyReward/mean": 0.775, "rewards/MveiAccuracyReward/std": 0.07071067690849304, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.5325000025331974, "rewards/MveiLLMConsistencyReward/std": 0.23568386361002922, "step": 120, "step_time": 78.01621430702508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00625, "completions/max_length": 273.2, "completions/mean_length": 197.70625, "completions/min_length": 168.55, "epoch": 0.09087604507451835, "frac_reward_zero_std": 0.15, "grad_norm": 1.2807616775097996, "kl": 0.011278474261052907, "learning_rate": 9.954771852494262e-07, "loss": 0.0004509925842285156, "reward": 1.7581249564886092, "reward_std": 0.2670029353350401, "rewards/MveiAccuracyReward/mean": 0.73125, "rewards/MveiAccuracyReward/std": 0.1075238049030304, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.5300000093877315, "rewards/MveiLLMConsistencyReward/std": 0.21534912884235383, "step": 125, "step_time": 83.60016283094883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.2, "completions/mean_length": 197.24375, "completions/min_length": 169.25, "epoch": 0.0945110868774991, "frac_reward_zero_std": 0.2, "grad_norm": 1.1290708719557516, "kl": 0.01565639809705317, "learning_rate": 9.946349959681802e-07, "loss": 0.0006264910101890564, "reward": 1.9900000154972077, "reward_std": 0.18564736619591712, "rewards/MveiAccuracyReward/mean": 0.9, "rewards/MveiAccuracyReward/std": 0.03535533845424652, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.5899999991059304, "rewards/MveiLLMConsistencyReward/std": 0.17345385067164898, "step": 130, "step_time": 76.15535179227591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 228.25, "completions/mean_length": 189.63125, "completions/min_length": 163.95, "epoch": 0.09814612868047982, "frac_reward_zero_std": 0.1, "grad_norm": 1.0272810605279092, "kl": 0.019541497249156235, "learning_rate": 9.937213622872584e-07, "loss": 0.0007814794778823852, "reward": 2.0331250190734864, "reward_std": 0.25848171301186085, "rewards/MveiAccuracyReward/mean": 0.88125, "rewards/MveiAccuracyReward/std": 0.0408231720328331, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.6549999989569187, "rewards/MveiLLMConsistencyReward/std": 0.22547860480844975, "step": 135, "step_time": 83.00242526158691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 228.95, "completions/mean_length": 194.59375, "completions/min_length": 163.6, "epoch": 0.10178117048346055, "frac_reward_zero_std": 0.1, "grad_norm": 1.2551395871215463, "kl": 0.0215371529571712, "learning_rate": 9.927364161706555e-07, "loss": 0.0008617550134658814, "reward": 1.8775000065565108, "reward_std": 0.2053673155605793, "rewards/MveiAccuracyReward/mean": 0.70625, "rewards/MveiAccuracyReward/std": 0.049022963643074034, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.6712500013411045, "rewards/MveiLLMConsistencyReward/std": 0.1611116800457239, "step": 140, "step_time": 84.6476160030812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.8, "completions/mean_length": 198.46875, "completions/min_length": 167.4, "epoch": 0.1054162122864413, "frac_reward_zero_std": 0.2, "grad_norm": 0.6100930813380608, "kl": 0.02497743829153478, "learning_rate": 9.916802998826365e-07, "loss": 0.0009988397359848022, "reward": 1.9999999761581422, "reward_std": 0.17097588442265987, "rewards/MveiAccuracyReward/mean": 0.88125, "rewards/MveiAccuracyReward/std": 0.025877460837364197, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.6187500074505806, "rewards/MveiLLMConsistencyReward/std": 0.1500001411885023, "step": 145, "step_time": 81.75836359225214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 237.1, "completions/mean_length": 195.95, "completions/min_length": 165.7, "epoch": 0.10905125408942203, "frac_reward_zero_std": 0.05, "grad_norm": 1.344354463877031, "kl": 0.027870298735797404, "learning_rate": 9.905531659671875e-07, "loss": 0.001115068793296814, "reward": 2.109999990463257, "reward_std": 0.2530000418424606, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.051754921674728394, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.6975000023841857, "rewards/MveiLLMConsistencyReward/std": 0.2356499671936035, "step": 150, "step_time": 74.11532084159553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 234.25, "completions/mean_length": 201.20625, "completions/min_length": 172.3, "epoch": 0.11268629589240277, "frac_reward_zero_std": 0.1, "grad_norm": 0.9416623465394275, "kl": 0.028954134788364172, "learning_rate": 9.89355177225984e-07, "loss": 0.0011582642793655396, "reward": 2.078749990463257, "reward_std": 0.2853988204151392, "rewards/MveiAccuracyReward/mean": 0.8875, "rewards/MveiAccuracyReward/std": 0.0667006328701973, "rewards/MveiFormatReward/mean": 0.9875, "rewards/MveiFormatReward/std": 0.03535533845424652, "rewards/MveiLLMConsistencyReward/mean": 0.6975000001490116, "rewards/MveiLLMConsistencyReward/std": 0.22299464270472527, "step": 155, "step_time": 74.52814088463784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 251.55, "completions/mean_length": 204.425, "completions/min_length": 172.3, "epoch": 0.1163213376953835, "frac_reward_zero_std": 0.1, "grad_norm": 1.3168534501650018, "kl": 0.03070834055542946, "learning_rate": 9.880865066948748e-07, "loss": 0.0012282460927963256, "reward": 2.093749976158142, "reward_std": 0.2688323128968477, "rewards/MveiAccuracyReward/mean": 0.875, "rewards/MveiAccuracyReward/std": 0.026726123690605164, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.7187500111758709, "rewards/MveiLLMConsistencyReward/std": 0.2422573085874319, "step": 160, "step_time": 75.55069470070302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.7, "completions/mean_length": 201.4125, "completions/min_length": 168.95, "epoch": 0.11995637949836423, "frac_reward_zero_std": 0.15, "grad_norm": 1.1277864654194623, "kl": 0.034754920098930595, "learning_rate": 9.867473376188896e-07, "loss": 0.001390466094017029, "reward": 2.108750009536743, "reward_std": 0.2290588106960058, "rewards/MveiAccuracyReward/mean": 0.85625, "rewards/MveiAccuracyReward/std": 0.06943259090185165, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.7525000058114528, "rewards/MveiLLMConsistencyReward/std": 0.17643247917294502, "step": 165, "step_time": 75.28065769933164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.55, "completions/mean_length": 204.075, "completions/min_length": 174.8, "epoch": 0.12359142130134497, "frac_reward_zero_std": 0.1, "grad_norm": 1.3030370810276026, "kl": 0.03521493105217814, "learning_rate": 9.853378634257708e-07, "loss": 0.0014090657234191895, "reward": 2.0225000351667406, "reward_std": 0.25071537867188454, "rewards/MveiAccuracyReward/mean": 0.76875, "rewards/MveiAccuracyReward/std": 0.09658813774585724, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.753749991953373, "rewards/MveiLLMConsistencyReward/std": 0.20075847208499908, "step": 170, "step_time": 80.61941720545292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.5, "completions/mean_length": 199.24375, "completions/min_length": 163.35, "epoch": 0.1272264631043257, "frac_reward_zero_std": 0.35, "grad_norm": 1.105875333905631, "kl": 0.03157349769026041, "learning_rate": 9.838582876980358e-07, "loss": 0.0012627072632312776, "reward": 2.086249992251396, "reward_std": 0.15274532735347748, "rewards/MveiAccuracyReward/mean": 0.83125, "rewards/MveiAccuracyReward/std": 0.0408231720328331, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.7550000041723252, "rewards/MveiLLMConsistencyReward/std": 0.126476688683033, "step": 175, "step_time": 70.45618718527257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.3, "completions/mean_length": 205.85, "completions/min_length": 171.55, "epoch": 0.13086150490730644, "frac_reward_zero_std": 0.25, "grad_norm": 1.0071507022324566, "kl": 0.0391253319568932, "learning_rate": 9.823088241435714e-07, "loss": 0.0015652745962142945, "reward": 2.213750022649765, "reward_std": 0.19908951558172702, "rewards/MveiAccuracyReward/mean": 0.86875, "rewards/MveiAccuracyReward/std": 0.044403792917728425, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8449999928474426, "rewards/MveiLLMConsistencyReward/std": 0.16771102957427503, "step": 180, "step_time": 80.93950719051062 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/mean_length": 201.80625, "completions/min_length": 169.9, "epoch": 0.13449654671028716, "frac_reward_zero_std": 0.1, "grad_norm": 1.3398320846525047, "kl": 0.03748435387387872, "learning_rate": 9.806896965647657e-07, "loss": 0.0014994919300079345, "reward": 2.163750022649765, "reward_std": 0.1716399945318699, "rewards/MveiAccuracyReward/mean": 0.85, "rewards/MveiAccuracyReward/std": 0.0, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8137499943375588, "rewards/MveiLLMConsistencyReward/std": 0.17163998819887638, "step": 185, "step_time": 75.46520342752338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.95, "completions/mean_length": 205.61875, "completions/min_length": 176.3, "epoch": 0.1381315885132679, "frac_reward_zero_std": 0.35, "grad_norm": 0.9078611397382459, "kl": 0.03967118514701724, "learning_rate": 9.790011388261832e-07, "loss": 0.0015863895416259766, "reward": 2.1424999982118607, "reward_std": 0.15116989873349668, "rewards/MveiAccuracyReward/mean": 0.85, "rewards/MveiAccuracyReward/std": 0.03535533845424652, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.7925000041723251, "rewards/MveiLLMConsistencyReward/std": 0.14499584771692753, "step": 190, "step_time": 72.58420044183731 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 236.55, "completions/mean_length": 204.53125, "completions/min_length": 179.4, "epoch": 0.14176663031624864, "frac_reward_zero_std": 0.15, "grad_norm": 1.1314213484288542, "kl": 0.03606957383453846, "learning_rate": 9.772433948207853e-07, "loss": 0.0014427393674850465, "reward": 2.2512499928474425, "reward_std": 0.13642010278999805, "rewards/MveiAccuracyReward/mean": 0.8375, "rewards/MveiAccuracyReward/std": 0.03535533845424652, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9137500062584877, "rewards/MveiLLMConsistencyReward/std": 0.10877882838249206, "step": 195, "step_time": 66.37393386811019 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.7, "completions/mean_length": 201.05625, "completions/min_length": 173.85, "epoch": 0.14540167211922936, "frac_reward_zero_std": 0.15, "grad_norm": 1.0024507743569822, "kl": 0.04934713859111071, "learning_rate": 9.754167184347025e-07, "loss": 0.0019738376140594482, "reward": 2.355000001192093, "reward_std": 0.17032154574990271, "rewards/MveiAccuracyReward/mean": 0.975, "rewards/MveiAccuracyReward/std": 0.046291005611419675, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8800000011920929, "rewards/MveiLLMConsistencyReward/std": 0.15183652900159358, "step": 200, "step_time": 70.0125329516828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00625, "completions/max_length": 281.15, "completions/mean_length": 212.16875, "completions/min_length": 172.35, "epoch": 0.1490367139222101, "frac_reward_zero_std": 0.4, "grad_norm": 1.00198513572539, "kl": 0.0418359481729567, "learning_rate": 9.73521373510564e-07, "loss": 0.0016731560230255128, "reward": 2.2337499856948853, "reward_std": 0.15645107291638852, "rewards/MveiAccuracyReward/mean": 0.8375, "rewards/MveiAccuracyReward/std": 0.049871626496315005, "rewards/MveiFormatReward/mean": 0.9875, "rewards/MveiFormatReward/std": 0.03535533845424652, "rewards/MveiLLMConsistencyReward/mean": 0.9025000005960464, "rewards/MveiLLMConsistencyReward/std": 0.11298311166465283, "step": 205, "step_time": 65.98886915110052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.6, "completions/mean_length": 206.9, "completions/min_length": 172.75, "epoch": 0.15267175572519084, "frac_reward_zero_std": 0.15, "grad_norm": 1.2981419714343267, "kl": 0.0480030654463917, "learning_rate": 9.715576338093886e-07, "loss": 0.0019203543663024903, "reward": 2.1325000047683718, "reward_std": 0.2264687493443489, "rewards/MveiAccuracyReward/mean": 0.85625, "rewards/MveiAccuracyReward/std": 0.06123279929161072, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.7762500017881393, "rewards/MveiLLMConsistencyReward/std": 0.1965201873332262, "step": 210, "step_time": 71.89682666286826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 230.15, "completions/mean_length": 194.93125, "completions/min_length": 167.65, "epoch": 0.1563067975281716, "frac_reward_zero_std": 0.45, "grad_norm": 0.06489088743601187, "kl": 0.04637978160753846, "learning_rate": 9.69525782971042e-07, "loss": 0.0018558710813522338, "reward": 2.3125000417232515, "reward_std": 0.11380274556577205, "rewards/MveiAccuracyReward/mean": 0.88125, "rewards/MveiAccuracyReward/std": 0.025877460837364197, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9312499910593033, "rewards/MveiLLMConsistencyReward/std": 0.0888317558914423, "step": 215, "step_time": 65.11796665452421 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.65, "completions/mean_length": 210.25625, "completions/min_length": 179.9, "epoch": 0.1599418393311523, "frac_reward_zero_std": 0.2, "grad_norm": 1.2863226714929106, "kl": 0.046673784218728545, "learning_rate": 9.6742611447327e-07, "loss": 0.0018677100539207458, "reward": 2.2906250178813936, "reward_std": 0.17234257869422437, "rewards/MveiAccuracyReward/mean": 0.925, "rewards/MveiAccuracyReward/std": 0.0667006328701973, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.86875, "rewards/MveiLLMConsistencyReward/std": 0.13413323648273945, "step": 220, "step_time": 72.660677921772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 250.25, "completions/mean_length": 201.9875, "completions/min_length": 168.8, "epoch": 0.16357688113413305, "frac_reward_zero_std": 0.25, "grad_norm": 1.2082426371244996, "kl": 0.046049478184431794, "learning_rate": 9.652589315893074e-07, "loss": 0.0018424808979034424, "reward": 2.238749998807907, "reward_std": 0.16750994548201562, "rewards/MveiAccuracyReward/mean": 0.84375, "rewards/MveiAccuracyReward/std": 0.05260358452796936, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8950000017881393, "rewards/MveiLLMConsistencyReward/std": 0.11923237554728985, "step": 225, "step_time": 75.52490624561906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.3, "completions/mean_length": 198.7375, "completions/min_length": 170.0, "epoch": 0.1672119229371138, "frac_reward_zero_std": 0.25, "grad_norm": 0.9236178019080928, "kl": 0.04613093789666891, "learning_rate": 9.630245473440742e-07, "loss": 0.0018455177545547486, "reward": 2.2575000166893004, "reward_std": 0.1700163245201111, "rewards/MveiAccuracyReward/mean": 0.86875, "rewards/MveiAccuracyReward/std": 0.06123279929161072, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8887499965727329, "rewards/MveiLLMConsistencyReward/std": 0.11767456233501435, "step": 230, "step_time": 69.37349548526109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 239.5, "completions/mean_length": 203.8125, "completions/min_length": 172.9, "epoch": 0.1708469647400945, "frac_reward_zero_std": 0.3, "grad_norm": 0.9845440231907046, "kl": 0.05328067895025015, "learning_rate": 9.607232844689633e-07, "loss": 0.0021315254271030426, "reward": 2.2650000005960464, "reward_std": 0.1732545718550682, "rewards/MveiAccuracyReward/mean": 0.89375, "rewards/MveiAccuracyReward/std": 0.05260358452796936, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8712499976158142, "rewards/MveiLLMConsistencyReward/std": 0.1268510937690735, "step": 235, "step_time": 70.66867304816842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.65, "completions/mean_length": 204.45625, "completions/min_length": 175.95, "epoch": 0.17448200654307525, "frac_reward_zero_std": 0.4, "grad_norm": 0.9490655950526524, "kl": 0.04163944832980633, "learning_rate": 9.583554753552244e-07, "loss": 0.0016658216714859008, "reward": 2.3737499892711638, "reward_std": 0.10231012664735317, "rewards/MveiAccuracyReward/mean": 0.9625, "rewards/MveiAccuracyReward/std": 0.051754921674728394, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9112500041723252, "rewards/MveiLLMConsistencyReward/std": 0.08913064040243626, "step": 240, "step_time": 76.31128382161259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.6, "completions/mean_length": 203.68125, "completions/min_length": 172.85, "epoch": 0.178117048346056, "frac_reward_zero_std": 0.25, "grad_norm": 1.0975017411676056, "kl": 0.05294170556589961, "learning_rate": 9.559214620059548e-07, "loss": 0.002118104696273804, "reward": 2.1712500035762785, "reward_std": 0.15000207386910916, "rewards/MveiAccuracyReward/mean": 0.79375, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8775000005960465, "rewards/MveiLLMConsistencyReward/std": 0.13570776283740998, "step": 245, "step_time": 74.231950693205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 241.6, "completions/mean_length": 204.40625, "completions/min_length": 175.65, "epoch": 0.1817520901490367, "frac_reward_zero_std": 0.4, "grad_norm": 0.610729868694356, "kl": 0.04313134495168924, "learning_rate": 9.534215959867009e-07, "loss": 0.001725362241268158, "reward": 2.318124997615814, "reward_std": 0.17341354563832284, "rewards/MveiAccuracyReward/mean": 0.9, "rewards/MveiAccuracyReward/std": 0.0667006328701973, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.9212500035762787, "rewards/MveiLLMConsistencyReward/std": 0.11625524051487446, "step": 250, "step_time": 69.31044210270048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.55, "completions/mean_length": 197.3875, "completions/min_length": 169.3, "epoch": 0.18538713195201745, "frac_reward_zero_std": 0.4, "grad_norm": 0.5939950616635872, "kl": 0.047632408887147905, "learning_rate": 9.508562383746782e-07, "loss": 0.0019053682684898377, "reward": 2.359999978542328, "reward_std": 0.09763411059975624, "rewards/MveiAccuracyReward/mean": 0.91875, "rewards/MveiAccuracyReward/std": 0.025877460837364197, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.941250005364418, "rewards/MveiLLMConsistencyReward/std": 0.07558601722121239, "step": 255, "step_time": 68.55646804682911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.65, "completions/mean_length": 199.56875, "completions/min_length": 170.8, "epoch": 0.1890221737549982, "frac_reward_zero_std": 0.35, "grad_norm": 1.4180883091743905, "kl": 0.04739540405571461, "learning_rate": 9.482257597066178e-07, "loss": 0.0018957346677780152, "reward": 2.372499978542328, "reward_std": 0.08922924064099788, "rewards/MveiAccuracyReward/mean": 0.94375, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.928750005364418, "rewards/MveiLLMConsistencyReward/std": 0.08588050492107868, "step": 260, "step_time": 68.8033996541053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 234.25, "completions/mean_length": 201.45625, "completions/min_length": 174.3, "epoch": 0.1926572155579789, "frac_reward_zero_std": 0.3, "grad_norm": 1.0886200490742217, "kl": 0.04904018035158515, "learning_rate": 9.455305399252467e-07, "loss": 0.0019615098834037782, "reward": 2.272499996423721, "reward_std": 0.18130445517599583, "rewards/MveiAccuracyReward/mean": 0.85625, "rewards/MveiAccuracyReward/std": 0.05260358452796936, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9162499994039536, "rewards/MveiLLMConsistencyReward/std": 0.14327077865600585, "step": 265, "step_time": 68.2446885280311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.45, "completions/mean_length": 210.21875, "completions/min_length": 180.85, "epoch": 0.19629225736095965, "frac_reward_zero_std": 0.25, "grad_norm": 1.2902017380177078, "kl": 0.05741015672683716, "learning_rate": 9.427709683244098e-07, "loss": 0.0022964745759963988, "reward": 2.280000013113022, "reward_std": 0.12487307526171207, "rewards/MveiAccuracyReward/mean": 0.88125, "rewards/MveiAccuracyReward/std": 0.025877460837364197, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8987499967217445, "rewards/MveiLLMConsistencyReward/std": 0.1037971641868353, "step": 270, "step_time": 83.63584203645587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 234.05, "completions/mean_length": 199.225, "completions/min_length": 169.35, "epoch": 0.1999272991639404, "frac_reward_zero_std": 0.3, "grad_norm": 1.076003319834389, "kl": 0.04970582015812397, "learning_rate": 9.3994744349284e-07, "loss": 0.0019883021712303163, "reward": 2.387499988079071, "reward_std": 0.12927508167922497, "rewards/MveiAccuracyReward/mean": 0.98125, "rewards/MveiAccuracyReward/std": 0.0408231720328331, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9062500029802323, "rewards/MveiLLMConsistencyReward/std": 0.09720045439898968, "step": 275, "step_time": 78.91972422488034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.75, "completions/mean_length": 209.825, "completions/min_length": 177.8, "epoch": 0.2035623409669211, "frac_reward_zero_std": 0.3, "grad_norm": 0.8446332609465718, "kl": 0.04920104956254363, "learning_rate": 9.370603732565874e-07, "loss": 0.001968240737915039, "reward": 2.2399999797344208, "reward_std": 0.21542084217071533, "rewards/MveiAccuracyReward/mean": 0.8625, "rewards/MveiAccuracyReward/std": 0.07490042448043824, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8775000035762787, "rewards/MveiLLMConsistencyReward/std": 0.15429446138441563, "step": 280, "step_time": 73.99278201758861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 234.55, "completions/mean_length": 198.93125, "completions/min_length": 169.65, "epoch": 0.20719738276990185, "frac_reward_zero_std": 0.3, "grad_norm": 1.1007666161198268, "kl": 0.050530897360295056, "learning_rate": 9.341101746201128e-07, "loss": 0.0020215705037117003, "reward": 2.188749998807907, "reward_std": 0.16432357504963874, "rewards/MveiAccuracyReward/mean": 0.78125, "rewards/MveiAccuracyReward/std": 0.07617851048707962, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9074999988079071, "rewards/MveiLLMConsistencyReward/std": 0.12360709756612778, "step": 285, "step_time": 76.36430631838739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.8, "completions/mean_length": 204.09375, "completions/min_length": 170.05, "epoch": 0.2108324245728826, "frac_reward_zero_std": 0.35, "grad_norm": 1.1056192913802685, "kl": 0.05204110499471426, "learning_rate": 9.310972737060565e-07, "loss": 0.002081531286239624, "reward": 2.2899999916553497, "reward_std": 0.1566681332886219, "rewards/MveiAccuracyReward/mean": 0.84375, "rewards/MveiAccuracyReward/std": 0.10435850620269775, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9462500005960465, "rewards/MveiLLMConsistencyReward/std": 0.07905281074345112, "step": 290, "step_time": 79.71511269360781 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 236.0, "completions/mean_length": 199.08125, "completions/min_length": 170.95, "epoch": 0.21446746637586334, "frac_reward_zero_std": 0.25, "grad_norm": 1.1126091469502415, "kl": 0.04745836248621345, "learning_rate": 9.280221056936899e-07, "loss": 0.0018991455435752869, "reward": 2.301874989271164, "reward_std": 0.11462128087878228, "rewards/MveiAccuracyReward/mean": 0.86875, "rewards/MveiAccuracyReward/std": 0.025877460837364197, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.9362500011920929, "rewards/MveiLLMConsistencyReward/std": 0.08925197273492813, "step": 295, "step_time": 71.66709435358644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.4, "completions/mean_length": 199.03125, "completions/min_length": 169.55, "epoch": 0.21810250817884405, "frac_reward_zero_std": 0.5, "grad_norm": 0.8923720692263183, "kl": 0.05621665976941585, "learning_rate": 9.248851147560584e-07, "loss": 0.002249002456665039, "reward": 2.3031250119209288, "reward_std": 0.13510362580418586, "rewards/MveiAccuracyReward/mean": 0.875, "rewards/MveiAccuracyReward/std": 0.07848104536533355, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.9312499940395356, "rewards/MveiLLMConsistencyReward/std": 0.08236783966422082, "step": 300, "step_time": 73.25305524095893 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 232.15, "completions/mean_length": 200.5125, "completions/min_length": 174.9, "epoch": 0.2217375499818248, "frac_reward_zero_std": 0.35, "grad_norm": 1.0861069784142643, "kl": 0.049988660216331485, "learning_rate": 9.216867539958258e-07, "loss": 0.0019994348287582396, "reward": 2.3087499916553496, "reward_std": 0.15251614190638066, "rewards/MveiAccuracyReward/mean": 0.89375, "rewards/MveiAccuracyReward/std": 0.06396867483854293, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9150000005960465, "rewards/MveiLLMConsistencyReward/std": 0.10046111457049847, "step": 305, "step_time": 69.34589745849371 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.45, "completions/mean_length": 214.9375, "completions/min_length": 183.35, "epoch": 0.22537259178480554, "frac_reward_zero_std": 0.25, "grad_norm": 0.6395480649768335, "kl": 0.04638021681457758, "learning_rate": 9.184274853798293e-07, "loss": 0.0018548578023910522, "reward": 2.2462499499320985, "reward_std": 0.19342792518436908, "rewards/MveiAccuracyReward/mean": 0.86875, "rewards/MveiAccuracyReward/std": 0.09069479852914811, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8775000095367431, "rewards/MveiLLMConsistencyReward/std": 0.1270702950656414, "step": 310, "step_time": 70.56883726418019 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.3, "completions/mean_length": 200.3, "completions/min_length": 172.2, "epoch": 0.22900763358778625, "frac_reward_zero_std": 0.55, "grad_norm": 0.5671343833841904, "kl": 0.0430236023850739, "learning_rate": 9.151077796723523e-07, "loss": 0.0017208054661750794, "reward": 2.3950000166893006, "reward_std": 0.11348208114504814, "rewards/MveiAccuracyReward/mean": 0.95625, "rewards/MveiAccuracyReward/std": 0.07216846644878387, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9387499958276748, "rewards/MveiLLMConsistencyReward/std": 0.06905492618680001, "step": 315, "step_time": 76.92188999690116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 222.5, "completions/mean_length": 194.75625, "completions/min_length": 167.65, "epoch": 0.232642675390767, "frac_reward_zero_std": 0.5, "grad_norm": 0.7242691547319087, "kl": 0.050981131847947836, "learning_rate": 9.117281163671293e-07, "loss": 0.0020397506654262544, "reward": 2.1637499928474426, "reward_std": 0.1011626586318016, "rewards/MveiAccuracyReward/mean": 0.7375, "rewards/MveiAccuracyReward/std": 0.05850084125995636, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9262500017881393, "rewards/MveiLLMConsistencyReward/std": 0.07830574735999107, "step": 320, "step_time": 72.86219071298838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 239.75, "completions/mean_length": 199.225, "completions/min_length": 169.05, "epoch": 0.23627771719374774, "frac_reward_zero_std": 0.2, "grad_norm": 1.2222972612905987, "kl": 0.05500196311622858, "learning_rate": 9.082889836180878e-07, "loss": 0.0022002458572387695, "reward": 2.2662500202655793, "reward_std": 0.1764941532164812, "rewards/MveiAccuracyReward/mean": 0.8375, "rewards/MveiAccuracyReward/std": 0.08984613567590713, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9287499934434891, "rewards/MveiLLMConsistencyReward/std": 0.10632340535521508, "step": 325, "step_time": 82.92368378974497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 226.8, "completions/mean_length": 199.74375, "completions/min_length": 174.25, "epoch": 0.23991275899672845, "frac_reward_zero_std": 0.45, "grad_norm": 0.8967205014832655, "kl": 0.047722284123301505, "learning_rate": 9.047908781688403e-07, "loss": 0.0019087165594100953, "reward": 2.3562499821186065, "reward_std": 0.09245501905679702, "rewards/MveiAccuracyReward/mean": 0.90625, "rewards/MveiAccuracyReward/std": 0.05260358452796936, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.95, "rewards/MveiLLMConsistencyReward/std": 0.06289107538759708, "step": 330, "step_time": 71.50175745785236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 239.85, "completions/mean_length": 203.53125, "completions/min_length": 174.35, "epoch": 0.2435478007997092, "frac_reward_zero_std": 0.55, "grad_norm": 0.5817438894302167, "kl": 0.04887179965153336, "learning_rate": 9.012343052809345e-07, "loss": 0.0019553512334823608, "reward": 2.4512499928474427, "reward_std": 0.07724135890603065, "rewards/MveiAccuracyReward/mean": 0.99375, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9575000017881393, "rewards/MveiLLMConsistencyReward/std": 0.06460157074034215, "step": 335, "step_time": 75.22226010747254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 244.2, "completions/mean_length": 206.1, "completions/min_length": 174.35, "epoch": 0.24718284260268994, "frac_reward_zero_std": 0.35, "grad_norm": 1.066274323490832, "kl": 0.04784417673945427, "learning_rate": 8.976197786608755e-07, "loss": 0.001913994550704956, "reward": 2.183124992251396, "reward_std": 0.21415966041386128, "rewards/MveiAccuracyReward/mean": 0.8375, "rewards/MveiAccuracyReward/std": 0.1165722593665123, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.848750001937151, "rewards/MveiLLMConsistencyReward/std": 0.12318442650139332, "step": 340, "step_time": 69.83715037330985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.15, "completions/mean_length": 201.46875, "completions/min_length": 174.45, "epoch": 0.25081788440567065, "frac_reward_zero_std": 0.3, "grad_norm": 1.221705888191254, "kl": 0.0555890429764986, "learning_rate": 8.939478203859252e-07, "loss": 0.0022241178900003432, "reward": 2.307500010728836, "reward_std": 0.1992207895964384, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.0974368005990982, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.894999997317791, "rewards/MveiLLMConsistencyReward/std": 0.11989384666085243, "step": 345, "step_time": 83.6939534559846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.55, "completions/mean_length": 201.30625, "completions/min_length": 173.6, "epoch": 0.2544529262086514, "frac_reward_zero_std": 0.4, "grad_norm": 1.0766859278224827, "kl": 0.04659942863509059, "learning_rate": 8.902189608286955e-07, "loss": 0.0018643148243427277, "reward": 2.2662499874830244, "reward_std": 0.13397842943668364, "rewards/MveiAccuracyReward/mean": 0.83125, "rewards/MveiAccuracyReward/std": 0.07617851048707962, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9350000053644181, "rewards/MveiLLMConsistencyReward/std": 0.06851454712450504, "step": 350, "step_time": 64.66634232662618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.9, "completions/mean_length": 201.6375, "completions/min_length": 175.4, "epoch": 0.25808796801163214, "frac_reward_zero_std": 0.1, "grad_norm": 1.2979092849263858, "kl": 0.04944706056267023, "learning_rate": 8.864337385805406e-07, "loss": 0.0019773632287979127, "reward": 2.3549999713897707, "reward_std": 0.1313345231115818, "rewards/MveiAccuracyReward/mean": 0.93125, "rewards/MveiAccuracyReward/std": 0.0408231720328331, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9237500101327896, "rewards/MveiLLMConsistencyReward/std": 0.1102304808795452, "step": 355, "step_time": 66.65488817803562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 232.4, "completions/mean_length": 198.9375, "completions/min_length": 165.75, "epoch": 0.2617230098146129, "frac_reward_zero_std": 0.5, "grad_norm": 0.5964610310424787, "kl": 0.050580203905701636, "learning_rate": 8.825927003737652e-07, "loss": 0.0020235240459442138, "reward": 2.2137500196695328, "reward_std": 0.14760382063686847, "rewards/MveiAccuracyReward/mean": 0.80625, "rewards/MveiAccuracyReward/std": 0.0925780937075615, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.907499996572733, "rewards/MveiLLMConsistencyReward/std": 0.07478510178625583, "step": 360, "step_time": 70.3552474875003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 239.4, "completions/mean_length": 206.4875, "completions/min_length": 179.8, "epoch": 0.2653580516175936, "frac_reward_zero_std": 0.3, "grad_norm": 1.0660771631170882, "kl": 0.05044060843065381, "learning_rate": 8.786964010026541e-07, "loss": 0.002017676830291748, "reward": 2.343749976158142, "reward_std": 0.12506617233157158, "rewards/MveiAccuracyReward/mean": 0.9, "rewards/MveiAccuracyReward/std": 0.07071067690849304, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9437500089406967, "rewards/MveiLLMConsistencyReward/std": 0.08580026850104332, "step": 365, "step_time": 77.13768363855779 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 244.9, "completions/mean_length": 208.29375, "completions/min_length": 180.6, "epoch": 0.2689930934205743, "frac_reward_zero_std": 0.3, "grad_norm": 0.8630473476120927, "kl": 0.05125547675415874, "learning_rate": 8.747454032433386e-07, "loss": 0.0020504653453826903, "reward": 2.267499989271164, "reward_std": 0.19351629428565503, "rewards/MveiAccuracyReward/mean": 0.8625, "rewards/MveiAccuracyReward/std": 0.13698188662528993, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9049999982118606, "rewards/MveiLLMConsistencyReward/std": 0.11784395799040795, "step": 370, "step_time": 65.95042751803994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 244.65, "completions/mean_length": 205.79375, "completions/min_length": 174.7, "epoch": 0.27262813522355506, "frac_reward_zero_std": 0.25, "grad_norm": 1.2324165165629464, "kl": 0.05260043805465102, "learning_rate": 8.707402777725103e-07, "loss": 0.002104008197784424, "reward": 2.3274999976158144, "reward_std": 0.1802256189286709, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.06208146214485168, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9150000005960465, "rewards/MveiLLMConsistencyReward/std": 0.13712091296911239, "step": 375, "step_time": 64.55796298235654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 228.65, "completions/mean_length": 199.41875, "completions/min_length": 171.9, "epoch": 0.2762631770265358, "frac_reward_zero_std": 0.3, "grad_norm": 0.06080417156751461, "kl": 0.055795913003385066, "learning_rate": 8.666816030849932e-07, "loss": 0.002232480049133301, "reward": 2.3762500047683717, "reward_std": 0.1070314772427082, "rewards/MveiAccuracyReward/mean": 0.94375, "rewards/MveiAccuracyReward/std": 0.049022963643074034, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9324999988079071, "rewards/MveiLLMConsistencyReward/std": 0.07924907505512238, "step": 380, "step_time": 63.71235692538321 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 251.4, "completions/mean_length": 198.98125, "completions/min_length": 169.25, "epoch": 0.27989821882951654, "frac_reward_zero_std": 0.55, "grad_norm": 1.1386499479267451, "kl": 0.04871965404599905, "learning_rate": 8.625699654101863e-07, "loss": 0.0019491560757160186, "reward": 2.441250002384186, "reward_std": 0.08268089033663273, "rewards/MveiAccuracyReward/mean": 0.975, "rewards/MveiAccuracyReward/std": 0.043555130064487454, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9662499994039535, "rewards/MveiLLMConsistencyReward/std": 0.05244523510336876, "step": 385, "step_time": 63.30250987969339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 228.25, "completions/mean_length": 194.51875, "completions/min_length": 167.4, "epoch": 0.2835332606324973, "frac_reward_zero_std": 0.5, "grad_norm": 0.9166107345765797, "kl": 0.056752162799239156, "learning_rate": 8.584059586273904e-07, "loss": 0.002271050214767456, "reward": 2.4256250143051146, "reward_std": 0.12021541744470596, "rewards/MveiAccuracyReward/mean": 0.9625, "rewards/MveiAccuracyReward/std": 0.0667006328701973, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.9662499964237213, "rewards/MveiLLMConsistencyReward/std": 0.06097796447575092, "step": 390, "step_time": 62.594334272667766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.25, "completions/mean_length": 202.8625, "completions/min_length": 174.25, "epoch": 0.28716830243547803, "frac_reward_zero_std": 0.25, "grad_norm": 1.2235684517287873, "kl": 0.05721313580870628, "learning_rate": 8.541901841800276e-07, "loss": 0.002288556098937988, "reward": 2.2050000011920927, "reward_std": 0.17486557103693484, "rewards/MveiAccuracyReward/mean": 0.7625, "rewards/MveiAccuracyReward/std": 0.12477205097675323, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9424999952316284, "rewards/MveiLLMConsistencyReward/std": 0.07683403715491295, "step": 395, "step_time": 63.35783043056726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.6, "completions/mean_length": 206.1375, "completions/min_length": 174.05, "epoch": 0.2908033442384587, "frac_reward_zero_std": 0.3, "grad_norm": 1.3283157079171533, "kl": 0.05699926046654582, "learning_rate": 8.499232509887711e-07, "loss": 0.0022801071405410766, "reward": 2.312500011920929, "reward_std": 0.15544865280389786, "rewards/MveiAccuracyReward/mean": 0.8625, "rewards/MveiAccuracyReward/std": 0.09342675656080246, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.95, "rewards/MveiLLMConsistencyReward/std": 0.07170455940067769, "step": 400, "step_time": 62.2062132999301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 220.8, "completions/mean_length": 190.46875, "completions/min_length": 162.9, "epoch": 0.29443838604143946, "frac_reward_zero_std": 0.5, "grad_norm": 0.5493762494591856, "kl": 0.05629263436421752, "learning_rate": 8.456057753635922e-07, "loss": 0.0022518396377563476, "reward": 2.3249999821186065, "reward_std": 0.10868542343378067, "rewards/MveiAccuracyReward/mean": 0.85625, "rewards/MveiAccuracyReward/std": 0.06943259090185165, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.96875, "rewards/MveiLLMConsistencyReward/std": 0.045820656791329385, "step": 405, "step_time": 64.3292087059468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 229.6, "completions/mean_length": 197.7375, "completions/min_length": 173.05, "epoch": 0.2980734278444202, "frac_reward_zero_std": 0.45, "grad_norm": 0.9019815140117805, "kl": 0.059184264950454236, "learning_rate": 8.412383809147421e-07, "loss": 0.0023672252893447874, "reward": 2.304999977350235, "reward_std": 0.11598805524408817, "rewards/MveiAccuracyReward/mean": 0.85, "rewards/MveiAccuracyReward/std": 0.0667006328701973, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9550000041723251, "rewards/MveiLLMConsistencyReward/std": 0.06418436132371426, "step": 410, "step_time": 65.96730939485133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 241.1, "completions/mean_length": 200.85, "completions/min_length": 168.2, "epoch": 0.30170846964740095, "frac_reward_zero_std": 0.55, "grad_norm": 0.872319679944253, "kl": 0.049759996309876445, "learning_rate": 8.368216984626787e-07, "loss": 0.0019900724291801453, "reward": 2.310000002384186, "reward_std": 0.09225410297513008, "rewards/MveiAccuracyReward/mean": 0.85625, "rewards/MveiAccuracyReward/std": 0.0408231720328331, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9537499994039536, "rewards/MveiLLMConsistencyReward/std": 0.05662188455462456, "step": 415, "step_time": 60.074932384863494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.95, "completions/mean_length": 208.09375, "completions/min_length": 179.6, "epoch": 0.3053435114503817, "frac_reward_zero_std": 0.3, "grad_norm": 1.0594661866937356, "kl": 0.06661088336259127, "learning_rate": 8.323563659469512e-07, "loss": 0.002664998173713684, "reward": 2.3474999964237213, "reward_std": 0.1417607918381691, "rewards/MveiAccuracyReward/mean": 0.89375, "rewards/MveiAccuracyReward/std": 0.08795892298221589, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9537500023841858, "rewards/MveiLLMConsistencyReward/std": 0.07330291867256164, "step": 420, "step_time": 71.8682894039899 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 225.5, "completions/mean_length": 197.33125, "completions/min_length": 173.6, "epoch": 0.30897855325336243, "frac_reward_zero_std": 0.2, "grad_norm": 1.4046658369688225, "kl": 0.055291316099464895, "learning_rate": 8.278430283340573e-07, "loss": 0.002212509512901306, "reward": 2.2975000202655793, "reward_std": 0.11464236341416836, "rewards/MveiAccuracyReward/mean": 0.85, "rewards/MveiAccuracyReward/std": 0.05850084125995636, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9474999934434891, "rewards/MveiLLMConsistencyReward/std": 0.07296968623995781, "step": 425, "step_time": 63.44632751494646 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 253.85, "completions/mean_length": 200.4625, "completions/min_length": 170.0, "epoch": 0.3126135950563432, "frac_reward_zero_std": 0.5, "grad_norm": 0.9073856219255105, "kl": 0.04947709441184998, "learning_rate": 8.232823375242847e-07, "loss": 0.0019794270396232606, "reward": 2.3512500166893004, "reward_std": 0.12073596306145191, "rewards/MveiAccuracyReward/mean": 0.9, "rewards/MveiAccuracyReward/std": 0.05850084125995636, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9512499958276749, "rewards/MveiLLMConsistencyReward/std": 0.0663618292659521, "step": 430, "step_time": 66.05351043827832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.8, "completions/mean_length": 203.58125, "completions/min_length": 174.95, "epoch": 0.31624863685932386, "frac_reward_zero_std": 0.55, "grad_norm": 0.5903155787768735, "kl": 0.057411357387900354, "learning_rate": 8.186749522575522e-07, "loss": 0.0022960007190704346, "reward": 2.278749966621399, "reward_std": 0.10757340379059314, "rewards/MveiAccuracyReward/mean": 0.825, "rewards/MveiAccuracyReward/std": 0.0667006328701973, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9537500023841858, "rewards/MveiLLMConsistencyReward/std": 0.06164647303521633, "step": 435, "step_time": 63.88984635472298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.95, "completions/mean_length": 201.425, "completions/min_length": 171.05, "epoch": 0.3198836786623046, "frac_reward_zero_std": 0.35, "grad_norm": 1.2139861427067402, "kl": 0.050447986274957654, "learning_rate": 8.140215380182616e-07, "loss": 0.002018173038959503, "reward": 2.4462499976158143, "reward_std": 0.104108402505517, "rewards/MveiAccuracyReward/mean": 0.98125, "rewards/MveiAccuracyReward/std": 0.0408231720328331, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9650000005960464, "rewards/MveiLLMConsistencyReward/std": 0.06714780479669571, "step": 440, "step_time": 58.520873974263665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 236.6, "completions/mean_length": 203.875, "completions/min_length": 176.65, "epoch": 0.32351872046528535, "frac_reward_zero_std": 0.45, "grad_norm": 1.1225455396596835, "kl": 0.0539106716401875, "learning_rate": 8.093227669391765e-07, "loss": 0.0021564602851867677, "reward": 2.3699999928474424, "reward_std": 0.11594206914305687, "rewards/MveiAccuracyReward/mean": 0.90625, "rewards/MveiAccuracyReward/std": 0.06943259090185165, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9637500017881393, "rewards/MveiLLMConsistencyReward/std": 0.05517797470092774, "step": 445, "step_time": 65.62710179761052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 241.7, "completions/mean_length": 204.7875, "completions/min_length": 177.05, "epoch": 0.3271537622682661, "frac_reward_zero_std": 0.4, "grad_norm": 0.9484777674770357, "kl": 0.05883749946951866, "learning_rate": 8.045793177043398e-07, "loss": 0.002353560924530029, "reward": 2.1237499833106996, "reward_std": 0.17556292489171027, "rewards/MveiAccuracyReward/mean": 0.74375, "rewards/MveiAccuracyReward/std": 0.06396867483854293, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8800000071525573, "rewards/MveiLLMConsistencyReward/std": 0.12470418699085713, "step": 450, "step_time": 60.537814708426595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 244.25, "completions/mean_length": 202.1625, "completions/min_length": 172.5, "epoch": 0.33078880407124683, "frac_reward_zero_std": 0.35, "grad_norm": 1.3318374564649673, "kl": 0.05984954619780183, "learning_rate": 7.997918754510466e-07, "loss": 0.0023946911096572876, "reward": 2.231249988079071, "reward_std": 0.15656850002706052, "rewards/MveiAccuracyReward/mean": 0.825, "rewards/MveiAccuracyReward/std": 0.043555130064487454, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.90625, "rewards/MveiLLMConsistencyReward/std": 0.1252195317298174, "step": 455, "step_time": 70.18436226099729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.6, "completions/mean_length": 191.96875, "completions/min_length": 164.25, "epoch": 0.3344238458742276, "frac_reward_zero_std": 0.7, "grad_norm": 0.5471439970028221, "kl": 0.05672888103872538, "learning_rate": 7.949611316708826e-07, "loss": 0.002269802987575531, "reward": 2.4125, "reward_std": 0.060308949649333955, "rewards/MveiAccuracyReward/mean": 0.94375, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.96875, "rewards/MveiLLMConsistencyReward/std": 0.046318995952606204, "step": 460, "step_time": 66.42560303509235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 228.5, "completions/mean_length": 196.05, "completions/min_length": 169.5, "epoch": 0.33805888767720826, "frac_reward_zero_std": 0.2, "grad_norm": 1.2116747042882259, "kl": 0.05652013067156077, "learning_rate": 7.900877841098465e-07, "loss": 0.0022601976990699766, "reward": 2.328749990463257, "reward_std": 0.1699168708175421, "rewards/MveiAccuracyReward/mean": 0.925, "rewards/MveiAccuracyReward/std": 0.07301712930202484, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9037500053644181, "rewards/MveiLLMConsistencyReward/std": 0.11530957035720349, "step": 465, "step_time": 64.05582083277405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.25, "completions/mean_length": 207.1, "completions/min_length": 174.7, "epoch": 0.341693929480189, "frac_reward_zero_std": 0.3, "grad_norm": 1.1597501308228664, "kl": 0.05327767413109541, "learning_rate": 7.85172536667569e-07, "loss": 0.002130722999572754, "reward": 2.252499985694885, "reward_std": 0.11413553021848202, "rewards/MveiAccuracyReward/mean": 0.83125, "rewards/MveiAccuracyReward/std": 0.025877460837364197, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9212500005960464, "rewards/MveiLLMConsistencyReward/std": 0.10005183294415473, "step": 470, "step_time": 66.18105836473406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 229.95, "completions/mean_length": 198.90625, "completions/min_length": 174.05, "epoch": 0.34532897128316975, "frac_reward_zero_std": 0.35, "grad_norm": 1.3500653569538534, "kl": 0.05451708110049367, "learning_rate": 7.802160992956419e-07, "loss": 0.0021812736988067625, "reward": 2.2687500178813935, "reward_std": 0.145114778727293, "rewards/MveiAccuracyReward/mean": 0.84375, "rewards/MveiAccuracyReward/std": 0.07617851048707962, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9249999955296516, "rewards/MveiLLMConsistencyReward/std": 0.08570018373429775, "step": 475, "step_time": 63.98813854046166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 236.9, "completions/mean_length": 195.975, "completions/min_length": 168.25, "epoch": 0.3489640130861505, "frac_reward_zero_std": 0.45, "grad_norm": 1.1991339535324204, "kl": 0.0536842804402113, "learning_rate": 7.75219187895074e-07, "loss": 0.0021476447582244873, "reward": 2.1912500083446504, "reward_std": 0.1769543681293726, "rewards/MveiAccuracyReward/mean": 0.75625, "rewards/MveiAccuracyReward/std": 0.12246951609849929, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9349999964237213, "rewards/MveiLLMConsistencyReward/std": 0.08697443492710591, "step": 480, "step_time": 66.7920372530818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.35, "completions/mean_length": 202.66875, "completions/min_length": 174.6, "epoch": 0.35259905488913124, "frac_reward_zero_std": 0.25, "grad_norm": 0.9707836906255748, "kl": 0.05712243542075157, "learning_rate": 7.701825242128869e-07, "loss": 0.002285104990005493, "reward": 2.202500009536743, "reward_std": 0.11030773371458054, "rewards/MveiAccuracyReward/mean": 0.80625, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.8962499976158143, "rewards/MveiLLMConsistencyReward/std": 0.11521266996860505, "step": 485, "step_time": 69.42208571247757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.3, "completions/mean_length": 213.5625, "completions/min_length": 178.05, "epoch": 0.356234096692112, "frac_reward_zero_std": 0.1, "grad_norm": 1.4659303026360888, "kl": 0.05164956394582987, "learning_rate": 7.651068357378676e-07, "loss": 0.0020667314529418945, "reward": 2.263750010728836, "reward_std": 0.20939640030264856, "rewards/MveiAccuracyReward/mean": 0.8375, "rewards/MveiAccuracyReward/std": 0.13971776217222215, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9262499958276749, "rewards/MveiLLMConsistencyReward/std": 0.09756103716790676, "step": 490, "step_time": 72.6419134542346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00625, "completions/max_length": 273.75, "completions/mean_length": 205.7625, "completions/min_length": 173.25, "epoch": 0.35986913849509267, "frac_reward_zero_std": 0.35, "grad_norm": 1.053758263654244, "kl": 0.05370880430564284, "learning_rate": 7.599928555954906e-07, "loss": 0.0021483659744262694, "reward": 2.309375, "reward_std": 0.09666542373597622, "rewards/MveiAccuracyReward/mean": 0.88125, "rewards/MveiAccuracyReward/std": 0.044403792917728425, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.9312499970197677, "rewards/MveiLLMConsistencyReward/std": 0.08384425416588784, "step": 495, "step_time": 65.41909863501787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 236.5, "completions/mean_length": 204.11875, "completions/min_length": 174.6, "epoch": 0.3635041802980734, "frac_reward_zero_std": 0.45, "grad_norm": 1.2068139413662464, "kl": 0.05257331561297178, "learning_rate": 7.548413224420258e-07, "loss": 0.0021028101444244387, "reward": 2.3625000059604644, "reward_std": 0.0931675311177969, "rewards/MveiAccuracyReward/mean": 0.9375, "rewards/MveiAccuracyReward/std": 0.03535533845424652, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9249999940395355, "rewards/MveiLLMConsistencyReward/std": 0.07031436897814274, "step": 500, "step_time": 64.24607281126082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 238.15, "completions/mean_length": 204.14375, "completions/min_length": 177.4, "epoch": 0.36713922210105415, "frac_reward_zero_std": 0.3, "grad_norm": 0.8831413852549667, "kl": 0.05326737640425563, "learning_rate": 7.49652980357849e-07, "loss": 0.0021309852600097656, "reward": 2.428124988079071, "reward_std": 0.119987802952528, "rewards/MveiAccuracyReward/mean": 0.9875, "rewards/MveiAccuracyReward/std": 0.023145502805709837, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.9437500029802323, "rewards/MveiLLMConsistencyReward/std": 0.09070673920214176, "step": 505, "step_time": 65.36522831656039 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 244.05, "completions/mean_length": 205.71875, "completions/min_length": 178.15, "epoch": 0.3707742639040349, "frac_reward_zero_std": 0.25, "grad_norm": 1.135326604621287, "kl": 0.05665736943483353, "learning_rate": 7.444285787399669e-07, "loss": 0.002267000079154968, "reward": 2.2599999964237214, "reward_std": 0.10505998097360134, "rewards/MveiAccuracyReward/mean": 0.83125, "rewards/MveiAccuracyReward/std": 0.025877460837364197, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9287499994039535, "rewards/MveiLLMConsistencyReward/std": 0.08056277297437191, "step": 510, "step_time": 68.83228598050773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 248.6, "completions/mean_length": 213.84375, "completions/min_length": 185.05, "epoch": 0.37440930570701564, "frac_reward_zero_std": 0.4, "grad_norm": 1.1027208729128795, "kl": 0.052945070713758466, "learning_rate": 7.391688721937763e-07, "loss": 0.0021178215742111207, "reward": 2.197500002384186, "reward_std": 0.1987775254994631, "rewards/MveiAccuracyReward/mean": 0.7625, "rewards/MveiAccuracyReward/std": 0.17149247974157333, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9350000053644181, "rewards/MveiLLMConsistencyReward/std": 0.05919569656252861, "step": 515, "step_time": 60.320371775701645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.45, "completions/mean_length": 203.0875, "completions/min_length": 177.1, "epoch": 0.3780443475099964, "frac_reward_zero_std": 0.25, "grad_norm": 1.2880362322197543, "kl": 0.05523567423224449, "learning_rate": 7.338746204240687e-07, "loss": 0.0022095203399658202, "reward": 2.2018749833106996, "reward_std": 0.18473673313856126, "rewards/MveiAccuracyReward/mean": 0.775, "rewards/MveiAccuracyReward/std": 0.1142658069729805, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.9300000041723251, "rewards/MveiLLMConsistencyReward/std": 0.07789790332317352, "step": 520, "step_time": 62.576503900438546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.25, "completions/mean_length": 205.99375, "completions/min_length": 175.75, "epoch": 0.3816793893129771, "frac_reward_zero_std": 0.3, "grad_norm": 1.1342890098182488, "kl": 0.05601891297847032, "learning_rate": 7.285465881253008e-07, "loss": 0.002241194248199463, "reward": 2.1987499952316285, "reward_std": 0.2141023449599743, "rewards/MveiAccuracyReward/mean": 0.775, "rewards/MveiAccuracyReward/std": 0.1672886312007904, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9237499982118607, "rewards/MveiLLMConsistencyReward/std": 0.09724985435605049, "step": 525, "step_time": 62.094839595258236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.65, "completions/mean_length": 205.05625, "completions/min_length": 170.15, "epoch": 0.3853144311159578, "frac_reward_zero_std": 0.45, "grad_norm": 1.2172909373527205, "kl": 0.048401630483567715, "learning_rate": 7.231855448711426e-07, "loss": 0.0019360661506652832, "reward": 2.323749989271164, "reward_std": 0.09478306397795677, "rewards/MveiAccuracyReward/mean": 0.875, "rewards/MveiAccuracyReward/std": 0.043555130064487454, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9487500011920929, "rewards/MveiLLMConsistencyReward/std": 0.06333404555916786, "step": 530, "step_time": 63.70999586209655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.25, "completions/mean_length": 200.71875, "completions/min_length": 172.4, "epoch": 0.38894947291893855, "frac_reward_zero_std": 0.35, "grad_norm": 0.9261509026675876, "kl": 0.05064096115529537, "learning_rate": 7.177922650033206e-07, "loss": 0.002025720477104187, "reward": 2.3612500011920927, "reward_std": 0.1679020505398512, "rewards/MveiAccuracyReward/mean": 0.925, "rewards/MveiAccuracyReward/std": 0.08984613567590713, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9362499952316284, "rewards/MveiLLMConsistencyReward/std": 0.09141329899430276, "step": 535, "step_time": 61.261403454467654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.1, "completions/mean_length": 204.66875, "completions/min_length": 173.7, "epoch": 0.3925845147219193, "frac_reward_zero_std": 0.35, "grad_norm": 0.7197223117139973, "kl": 0.05635623093694449, "learning_rate": 7.123675275197738e-07, "loss": 0.0022535800933837892, "reward": 2.308749955892563, "reward_std": 0.15509552210569383, "rewards/MveiAccuracyReward/mean": 0.88125, "rewards/MveiAccuracyReward/std": 0.10290463417768478, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9275000095367432, "rewards/MveiLLMConsistencyReward/std": 0.06624302230775356, "step": 540, "step_time": 57.23934747502208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 223.7, "completions/mean_length": 193.38125, "completions/min_length": 168.8, "epoch": 0.39621955652490004, "frac_reward_zero_std": 0.35, "grad_norm": 1.2111291692233497, "kl": 0.04909519180655479, "learning_rate": 7.069121159621363e-07, "loss": 0.0019636735320091247, "reward": 2.3212499558925628, "reward_std": 0.0717705175280571, "rewards/MveiAccuracyReward/mean": 0.85625, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9650000125169754, "rewards/MveiLLMConsistencyReward/std": 0.05409285649657249, "step": 545, "step_time": 57.13765530437231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.7, "completions/mean_length": 201.70625, "completions/min_length": 168.6, "epoch": 0.3998545983278808, "frac_reward_zero_std": 0.5, "grad_norm": 0.9955385708676731, "kl": 0.04794792728498578, "learning_rate": 7.014268183025631e-07, "loss": 0.0019185051321983337, "reward": 2.3812500119209288, "reward_std": 0.1366258893162012, "rewards/MveiAccuracyReward/mean": 0.925, "rewards/MveiAccuracyReward/std": 0.09385617971420288, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9562499970197678, "rewards/MveiLLMConsistencyReward/std": 0.06136959791183472, "step": 550, "step_time": 62.35990489758551 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.7, "completions/mean_length": 213.225, "completions/min_length": 178.2, "epoch": 0.4034896401308615, "frac_reward_zero_std": 0.25, "grad_norm": 1.3799567537012887, "kl": 0.059047107398509976, "learning_rate": 6.959124268299166e-07, "loss": 0.002361568808555603, "reward": 2.2499999403953552, "reward_std": 0.12058468423783779, "rewards/MveiAccuracyReward/mean": 0.81875, "rewards/MveiAccuracyReward/std": 0.06123279929161072, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9312500178813934, "rewards/MveiLLMConsistencyReward/std": 0.07831006348133088, "step": 555, "step_time": 63.79237565882504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.45, "completions/mean_length": 209.0125, "completions/min_length": 178.9, "epoch": 0.4071246819338422, "frac_reward_zero_std": 0.2, "grad_norm": 1.0840077239241812, "kl": 0.0486101808026433, "learning_rate": 6.903697380353298e-07, "loss": 0.0019450187683105469, "reward": 2.4093749940395357, "reward_std": 0.11802061796188354, "rewards/MveiAccuracyReward/mean": 0.9625, "rewards/MveiAccuracyReward/std": 0.023145502805709837, "rewards/MveiFormatReward/mean": 0.98125, "rewards/MveiFormatReward/std": 0.05303300768136978, "rewards/MveiLLMConsistencyReward/mean": 0.9562500029802322, "rewards/MveiLLMConsistencyReward/std": 0.07798741534352302, "step": 560, "step_time": 60.060036178678274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 238.1, "completions/mean_length": 198.9625, "completions/min_length": 169.45, "epoch": 0.41075972373682296, "frac_reward_zero_std": 0.25, "grad_norm": 1.1811123278584712, "kl": 0.05753252692520618, "learning_rate": 6.847995524971619e-07, "loss": 0.0023005783557891847, "reward": 2.263750010728836, "reward_std": 0.13544689528644085, "rewards/MveiAccuracyReward/mean": 0.80625, "rewards/MveiAccuracyReward/std": 0.0757490873336792, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9574999928474426, "rewards/MveiLLMConsistencyReward/std": 0.0672094389796257, "step": 565, "step_time": 64.18464987687767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.2, "completions/mean_length": 204.38125, "completions/min_length": 175.4, "epoch": 0.4143947655398037, "frac_reward_zero_std": 0.45, "grad_norm": 0.5912123187155788, "kl": 0.05996296042576432, "learning_rate": 6.792026747653643e-07, "loss": 0.0023984819650650024, "reward": 2.26499999165535, "reward_std": 0.050737760961055756, "rewards/MveiAccuracyReward/mean": 0.8, "rewards/MveiAccuracyReward/std": 0.0, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9650000005960464, "rewards/MveiLLMConsistencyReward/std": 0.05073775686323643, "step": 570, "step_time": 58.63816139064729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 229.5, "completions/mean_length": 196.88125, "completions/min_length": 166.7, "epoch": 0.41802980734278444, "frac_reward_zero_std": 0.5, "grad_norm": 1.0153067081656428, "kl": 0.05443270392715931, "learning_rate": 6.735799132452719e-07, "loss": 0.0021768227219581605, "reward": 2.431249976158142, "reward_std": 0.09589770957827567, "rewards/MveiAccuracyReward/mean": 0.95625, "rewards/MveiAccuracyReward/std": 0.06754929572343826, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9750000029802323, "rewards/MveiLLMConsistencyReward/std": 0.050441878661513326, "step": 575, "step_time": 59.038442390412094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.3, "completions/mean_length": 205.7625, "completions/min_length": 180.05, "epoch": 0.4216648491457652, "frac_reward_zero_std": 0.15, "grad_norm": 1.134960103349415, "kl": 0.05283217802643776, "learning_rate": 6.679320800808386e-07, "loss": 0.0021133676171302797, "reward": 2.2875, "reward_std": 0.1303585585206747, "rewards/MveiAccuracyReward/mean": 0.8625, "rewards/MveiAccuracyReward/std": 0.043555130064487454, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9250000029802322, "rewards/MveiLLMConsistencyReward/std": 0.09563095308840275, "step": 580, "step_time": 63.406537940725684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 237.45, "completions/mean_length": 200.23125, "completions/min_length": 173.05, "epoch": 0.42529989094874593, "frac_reward_zero_std": 0.3, "grad_norm": 0.8157496467150972, "kl": 0.06471217218786478, "learning_rate": 6.622599910373317e-07, "loss": 0.002588161826133728, "reward": 2.372499966621399, "reward_std": 0.0735352661460638, "rewards/MveiAccuracyReward/mean": 0.90625, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9662500083446502, "rewards/MveiLLMConsistencyReward/std": 0.06310496106743813, "step": 585, "step_time": 65.69516163021326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.3, "completions/mean_length": 201.24375, "completions/min_length": 172.7, "epoch": 0.42893493275172667, "frac_reward_zero_std": 0.4, "grad_norm": 1.237601541269308, "kl": 0.056501855980604886, "learning_rate": 6.565644653835053e-07, "loss": 0.002260762453079224, "reward": 2.410000002384186, "reward_std": 0.13696115352213384, "rewards/MveiAccuracyReward/mean": 0.975, "rewards/MveiAccuracyReward/std": 0.043555130064487454, "rewards/MveiFormatReward/mean": 0.9875, "rewards/MveiFormatReward/std": 0.03535533845424652, "rewards/MveiLLMConsistencyReward/mean": 0.9412499994039536, "rewards/MveiLLMConsistencyReward/std": 0.0803681381046772, "step": 590, "step_time": 61.110890195518735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 234.9, "completions/mean_length": 202.1, "completions/min_length": 175.1, "epoch": 0.43256997455470736, "frac_reward_zero_std": 0.5, "grad_norm": 0.08556000264668923, "kl": 0.06673226188868284, "learning_rate": 6.508463257732644e-07, "loss": 0.0026691555976867676, "reward": 2.352500003576279, "reward_std": 0.07930894531309604, "rewards/MveiAccuracyReward/mean": 0.8875, "rewards/MveiAccuracyReward/std": 0.03535533845424652, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9649999976158142, "rewards/MveiLLMConsistencyReward/std": 0.04610480107367039, "step": 595, "step_time": 63.12421229444444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.45, "completions/mean_length": 199.64375, "completions/min_length": 172.9, "epoch": 0.4362050163576881, "frac_reward_zero_std": 0.4, "grad_norm": 1.0473929863133022, "kl": 0.050745103228837254, "learning_rate": 6.451063981268433e-07, "loss": 0.0020303398370742796, "reward": 2.468750023841858, "reward_std": 0.05933799929916859, "rewards/MveiAccuracyReward/mean": 1.0, "rewards/MveiAccuracyReward/std": 0.0, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9687499940395355, "rewards/MveiLLMConsistencyReward/std": 0.05933800302445889, "step": 600, "step_time": 66.82194680832326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.95, "completions/mean_length": 201.2375, "completions/min_length": 174.6, "epoch": 0.43984005816066885, "frac_reward_zero_std": 0.2, "grad_norm": 0.9629096188446689, "kl": 0.059405024908483026, "learning_rate": 6.393455115115104e-07, "loss": 0.0023769378662109376, "reward": 2.2493750154972076, "reward_std": 0.2034198261797428, "rewards/MveiAccuracyReward/mean": 0.825, "rewards/MveiAccuracyReward/std": 0.12878209501504898, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.9274999976158143, "rewards/MveiLLMConsistencyReward/std": 0.09270746670663357, "step": 605, "step_time": 61.957865204662085 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.15, "completions/mean_length": 206.30625, "completions/min_length": 176.05, "epoch": 0.4434750999636496, "frac_reward_zero_std": 0.3, "grad_norm": 1.1432314216345052, "kl": 0.059602943155914544, "learning_rate": 6.335644980218184e-07, "loss": 0.0023843377828598024, "reward": 2.223749977350235, "reward_std": 0.21208821050822735, "rewards/MveiAccuracyReward/mean": 0.7875, "rewards/MveiAccuracyReward/std": 0.1483469769358635, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9362499982118606, "rewards/MveiLLMConsistencyReward/std": 0.10256377197802066, "step": 610, "step_time": 61.47646673768759 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 227.55, "completions/mean_length": 197.175, "completions/min_length": 167.75, "epoch": 0.44711014176663033, "frac_reward_zero_std": 0.2, "grad_norm": 1.2303025699265895, "kl": 0.05380919221788645, "learning_rate": 6.277641926594185e-07, "loss": 0.0021525368094444273, "reward": 2.3424999892711638, "reward_std": 0.11703605018556118, "rewards/MveiAccuracyReward/mean": 0.9, "rewards/MveiAccuracyReward/std": 0.03535533845424652, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9425000071525573, "rewards/MveiLLMConsistencyReward/std": 0.08437751457095147, "step": 615, "step_time": 57.86312923617661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 236.45, "completions/mean_length": 198.2, "completions/min_length": 168.55, "epoch": 0.4507451835696111, "frac_reward_zero_std": 0.4, "grad_norm": 0.6999681934447477, "kl": 0.05240498147904873, "learning_rate": 6.219454332124531e-07, "loss": 0.0020967841148376463, "reward": 2.39375, "reward_std": 0.14114195108413696, "rewards/MveiAccuracyReward/mean": 0.95625, "rewards/MveiAccuracyReward/std": 0.09932401329278946, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9375, "rewards/MveiLLMConsistencyReward/std": 0.06857122667133808, "step": 620, "step_time": 58.16082239821553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 234.1, "completions/mean_length": 202.7625, "completions/min_length": 176.9, "epoch": 0.45438022537259176, "frac_reward_zero_std": 0.5, "grad_norm": 1.1825385245122997, "kl": 0.050347439106553794, "learning_rate": 6.161090601345476e-07, "loss": 0.0020137086510658266, "reward": 2.3837499916553497, "reward_std": 0.09976745806634427, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.043555130064487454, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9712500005960465, "rewards/MveiLLMConsistencyReward/std": 0.05759679488837719, "step": 625, "step_time": 63.5189865604043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 238.2, "completions/mean_length": 199.525, "completions/min_length": 173.85, "epoch": 0.4580152671755725, "frac_reward_zero_std": 0.55, "grad_norm": 0.6265346042375116, "kl": 0.04690617267042398, "learning_rate": 6.102559164234163e-07, "loss": 0.0018758177757263184, "reward": 2.4387499928474425, "reward_std": 0.07619544602930546, "rewards/MveiAccuracyReward/mean": 0.98125, "rewards/MveiAccuracyReward/std": 0.0408231720328331, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9575000017881393, "rewards/MveiLLMConsistencyReward/std": 0.046797850355505946, "step": 630, "step_time": 58.02576370835304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 229.15, "completions/mean_length": 197.35, "completions/min_length": 170.8, "epoch": 0.46165030897855325, "frac_reward_zero_std": 0.6, "grad_norm": 0.5489047862993448, "kl": 0.05437676999717951, "learning_rate": 6.043868474990996e-07, "loss": 0.0021751759573817252, "reward": 2.4262499809265137, "reward_std": 0.07427813299000263, "rewards/MveiAccuracyReward/mean": 0.95, "rewards/MveiAccuracyReward/std": 0.046291005611419675, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9762500077486038, "rewards/MveiLLMConsistencyReward/std": 0.04104428850114346, "step": 635, "step_time": 64.33075705319644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 237.75, "completions/mean_length": 200.06875, "completions/min_length": 168.8, "epoch": 0.465285350781534, "frac_reward_zero_std": 0.6, "grad_norm": 0.6484025378502892, "kl": 0.060050109215080735, "learning_rate": 5.985027010818552e-07, "loss": 0.0024017333984375, "reward": 2.381249988079071, "reward_std": 0.06976461969316006, "rewards/MveiAccuracyReward/mean": 0.925, "rewards/MveiAccuracyReward/std": 0.026726123690605164, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.95625, "rewards/MveiLLMConsistencyReward/std": 0.05022066310048103, "step": 640, "step_time": 60.223006937652826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 222.95, "completions/mean_length": 193.0, "completions/min_length": 165.75, "epoch": 0.46892039258451473, "frac_reward_zero_std": 0.15, "grad_norm": 1.276040088015636, "kl": 0.06414886265993118, "learning_rate": 5.926043270697123e-07, "loss": 0.002565912902355194, "reward": 2.211249965429306, "reward_std": 0.12579258158802986, "rewards/MveiAccuracyReward/mean": 0.80625, "rewards/MveiAccuracyReward/std": 0.06396867483854293, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9050000071525574, "rewards/MveiLLMConsistencyReward/std": 0.09588906839489937, "step": 645, "step_time": 67.38408568166196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 231.95, "completions/mean_length": 196.225, "completions/min_length": 166.15, "epoch": 0.4725554343874955, "frac_reward_zero_std": 0.3, "grad_norm": 0.633168537470746, "kl": 0.059809874184429644, "learning_rate": 5.866925774157148e-07, "loss": 0.002393084764480591, "reward": 2.3537499964237214, "reward_std": 0.13260694704949855, "rewards/MveiAccuracyReward/mean": 0.9, "rewards/MveiAccuracyReward/std": 0.0816463440656662, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9537500023841858, "rewards/MveiLLMConsistencyReward/std": 0.06922099255025387, "step": 650, "step_time": 64.30584721565246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 241.55, "completions/mean_length": 203.3125, "completions/min_length": 175.65, "epoch": 0.47619047619047616, "frac_reward_zero_std": 0.45, "grad_norm": 1.0780184778390836, "kl": 0.05320450058206916, "learning_rate": 5.807683060048658e-07, "loss": 0.0021285176277160644, "reward": 2.3974999845027924, "reward_std": 0.06946404278278351, "rewards/MveiAccuracyReward/mean": 0.94375, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9537500023841858, "rewards/MveiLLMConsistencyReward/std": 0.05438485443592071, "step": 655, "step_time": 61.625752969831225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 232.2, "completions/mean_length": 195.99375, "completions/min_length": 168.4, "epoch": 0.4798255179934569, "frac_reward_zero_std": 0.6, "grad_norm": 0.8238856197498328, "kl": 0.053611588478088376, "learning_rate": 5.748323685307936e-07, "loss": 0.002144557237625122, "reward": 2.4399999916553496, "reward_std": 0.07517711184918881, "rewards/MveiAccuracyReward/mean": 0.96875, "rewards/MveiAccuracyReward/std": 0.044403792917728425, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9712500035762787, "rewards/MveiLLMConsistencyReward/std": 0.04230758845806122, "step": 660, "step_time": 63.852589287608865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 222.2, "completions/mean_length": 195.25, "completions/min_length": 169.35, "epoch": 0.48346055979643765, "frac_reward_zero_std": 0.4, "grad_norm": 0.9303723416284634, "kl": 0.05216963700950146, "learning_rate": 5.68885622372157e-07, "loss": 0.0020866133272647858, "reward": 2.4524999856948853, "reward_std": 0.07636669464409351, "rewards/MveiAccuracyReward/mean": 1.0, "rewards/MveiAccuracyReward/std": 0.0, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9525000035762787, "rewards/MveiLLMConsistencyReward/std": 0.07636669613420963, "step": 665, "step_time": 63.11184250563383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 224.35, "completions/mean_length": 194.8625, "completions/min_length": 169.7, "epoch": 0.4870956015994184, "frac_reward_zero_std": 0.25, "grad_norm": 1.1743371080404714, "kl": 0.06406292896717787, "learning_rate": 5.629289264688061e-07, "loss": 0.002563074231147766, "reward": 2.348749989271164, "reward_std": 0.14019065350294113, "rewards/MveiAccuracyReward/mean": 0.89375, "rewards/MveiAccuracyReward/std": 0.08795892298221589, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9550000041723251, "rewards/MveiLLMConsistencyReward/std": 0.06603162772953511, "step": 670, "step_time": 180.70229631587864 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.2, "completions/mean_length": 203.6125, "completions/min_length": 174.4, "epoch": 0.49073064340239914, "frac_reward_zero_std": 0.3, "grad_norm": 0.8537166656005972, "kl": 0.05742366947233677, "learning_rate": 5.569631411977183e-07, "loss": 0.0022970378398895265, "reward": 2.3449999690055847, "reward_std": 0.11332251727581025, "rewards/MveiAccuracyReward/mean": 0.88125, "rewards/MveiAccuracyReward/std": 0.06123279929161072, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.963750010728836, "rewards/MveiLLMConsistencyReward/std": 0.054786515235900876, "step": 675, "step_time": 60.93428806103766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.75, "completions/mean_length": 199.125, "completions/min_length": 166.9, "epoch": 0.4943656852053799, "frac_reward_zero_std": 0.4, "grad_norm": 0.6826603397734794, "kl": 0.05021212715655565, "learning_rate": 5.509891282487275e-07, "loss": 0.0020084142684936523, "reward": 2.4324999928474424, "reward_std": 0.09412019178271294, "rewards/MveiAccuracyReward/mean": 0.98125, "rewards/MveiAccuracyReward/std": 0.0408231720328331, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9512500017881393, "rewards/MveiLLMConsistencyReward/std": 0.06251149512827396, "step": 680, "step_time": 59.32948975674808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 234.2, "completions/mean_length": 201.675, "completions/min_length": 172.55, "epoch": 0.4980007270083606, "frac_reward_zero_std": 0.4, "grad_norm": 0.9879158907430384, "kl": 0.059967555291950705, "learning_rate": 5.450077505000612e-07, "loss": 0.002398298680782318, "reward": 2.2287499845027923, "reward_std": 0.13407860919833184, "rewards/MveiAccuracyReward/mean": 0.81875, "rewards/MveiAccuracyReward/std": 0.06396867483854293, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9100000038743019, "rewards/MveiLLMConsistencyReward/std": 0.08820019215345383, "step": 685, "step_time": 60.08720072694123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 231.8, "completions/mean_length": 201.78125, "completions/min_length": 176.0, "epoch": 0.5016357688113413, "frac_reward_zero_std": 0.25, "grad_norm": 0.834265898422456, "kl": 0.05460627768188715, "learning_rate": 5.390198718937091e-07, "loss": 0.002184931933879852, "reward": 2.2975000143051147, "reward_std": 0.19341775104403497, "rewards/MveiAccuracyReward/mean": 0.8625, "rewards/MveiAccuracyReward/std": 0.12015288025140762, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9349999994039535, "rewards/MveiLLMConsistencyReward/std": 0.10299718491733074, "step": 690, "step_time": 64.25981214381754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.05, "completions/mean_length": 198.2875, "completions/min_length": 170.55, "epoch": 0.5052708106143221, "frac_reward_zero_std": 0.35, "grad_norm": 1.2727083271601063, "kl": 0.05565820280462504, "learning_rate": 5.330263573106357e-07, "loss": 0.0022266149520874024, "reward": 2.341250002384186, "reward_std": 0.117528111115098, "rewards/MveiAccuracyReward/mean": 0.89375, "rewards/MveiAccuracyReward/std": 0.06943259090185165, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9474999994039536, "rewards/MveiLLMConsistencyReward/std": 0.07141387052834033, "step": 695, "step_time": 180.17640886865556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.1, "completions/mean_length": 198.4875, "completions/min_length": 170.75, "epoch": 0.5089058524173028, "frac_reward_zero_std": 0.4, "grad_norm": 1.1063117753508258, "kl": 0.05431645596399903, "learning_rate": 5.270280724458579e-07, "loss": 0.0021736010909080505, "reward": 2.364999997615814, "reward_std": 0.11570962965488434, "rewards/MveiAccuracyReward/mean": 0.9, "rewards/MveiAccuracyReward/std": 0.07028125375509262, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9649999976158142, "rewards/MveiLLMConsistencyReward/std": 0.05579419694840908, "step": 700, "step_time": 62.90832507573068 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 227.4, "completions/mean_length": 201.525, "completions/min_length": 174.3, "epoch": 0.5125408942202835, "frac_reward_zero_std": 0.5, "grad_norm": 0.9227917459769821, "kl": 0.050359622575342655, "learning_rate": 5.210258836834061e-07, "loss": 0.0020142778754234313, "reward": 2.4349999904632567, "reward_std": 0.056512949988245964, "rewards/MveiAccuracyReward/mean": 0.95625, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9787500023841857, "rewards/MveiLLMConsistencyReward/std": 0.04191595278680325, "step": 705, "step_time": 65.42733619399368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 236.35, "completions/mean_length": 201.425, "completions/min_length": 170.75, "epoch": 0.5161759360232643, "frac_reward_zero_std": 0.55, "grad_norm": 1.1155377821534105, "kl": 0.050790361035615204, "learning_rate": 5.150206579711841e-07, "loss": 0.0020315080881118776, "reward": 2.3724999964237212, "reward_std": 0.1378706492483616, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.12015288025140762, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9599999964237214, "rewards/MveiLLMConsistencyReward/std": 0.05531466566026211, "step": 710, "step_time": 56.96984634622932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.2, "completions/mean_length": 207.6375, "completions/min_length": 181.4, "epoch": 0.519810977826245, "frac_reward_zero_std": 0.25, "grad_norm": 1.5449226412042787, "kl": 0.05760031826794147, "learning_rate": 5.090132626957483e-07, "loss": 0.002303384244441986, "reward": 2.3712499737739563, "reward_std": 0.1599471341818571, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.10205597132444381, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9587500065565109, "rewards/MveiLLMConsistencyReward/std": 0.07709407061338425, "step": 715, "step_time": 61.04374643303454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 230.85, "completions/mean_length": 197.825, "completions/min_length": 171.25, "epoch": 0.5234460196292258, "frac_reward_zero_std": 0.5, "grad_norm": 1.2051537027260706, "kl": 0.04487507436424494, "learning_rate": 5.030045655570239e-07, "loss": 0.001795440912246704, "reward": 2.443750023841858, "reward_std": 0.07350295260548592, "rewards/MveiAccuracyReward/mean": 0.99375, "rewards/MveiAccuracyReward/std": 0.01767766922712326, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9499999940395355, "rewards/MveiLLMConsistencyReward/std": 0.06459763720631599, "step": 720, "step_time": 61.92764515392482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 230.05, "completions/mean_length": 198.3125, "completions/min_length": 175.1, "epoch": 0.5270810614322065, "frac_reward_zero_std": 0.4, "grad_norm": 0.7604026219628306, "kl": 0.05456447992473841, "learning_rate": 4.96995434442976e-07, "loss": 0.0021826371550559996, "reward": 2.3399999678134917, "reward_std": 0.10628600828349591, "rewards/MveiAccuracyReward/mean": 0.90625, "rewards/MveiAccuracyReward/std": 0.06754929572343826, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.933750008046627, "rewards/MveiLLMConsistencyReward/std": 0.06852892078459263, "step": 725, "step_time": 56.75429000556469 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.6, "completions/mean_length": 201.1125, "completions/min_length": 172.55, "epoch": 0.5307161032351873, "frac_reward_zero_std": 0.35, "grad_norm": 1.0330584260741134, "kl": 0.05688001066446304, "learning_rate": 4.909867373042517e-07, "loss": 0.0022754698991775514, "reward": 2.2087499797344208, "reward_std": 0.14862833879888057, "rewards/MveiAccuracyReward/mean": 0.8, "rewards/MveiAccuracyReward/std": 0.09258201122283935, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.908750006556511, "rewards/MveiLLMConsistencyReward/std": 0.07889666929841041, "step": 730, "step_time": 64.73999388627708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 232.65, "completions/mean_length": 196.98125, "completions/min_length": 169.45, "epoch": 0.5343511450381679, "frac_reward_zero_std": 0.5, "grad_norm": 1.1474298194768007, "kl": 0.05623842403292656, "learning_rate": 4.84979342028816e-07, "loss": 0.0022498369216918945, "reward": 2.4000000238418577, "reward_std": 0.09364267885684967, "rewards/MveiAccuracyReward/mean": 0.9375, "rewards/MveiAccuracyReward/std": 0.03535533845424652, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9624999940395356, "rewards/MveiLLMConsistencyReward/std": 0.06328011900186539, "step": 735, "step_time": 177.2352121386677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 229.8, "completions/mean_length": 199.1375, "completions/min_length": 170.8, "epoch": 0.5379861868411486, "frac_reward_zero_std": 0.7, "grad_norm": 0.6896036376003111, "kl": 0.051563126314431426, "learning_rate": 4.789741163165938e-07, "loss": 0.002062559127807617, "reward": 2.3737499892711638, "reward_std": 0.05844749584794044, "rewards/MveiAccuracyReward/mean": 0.90625, "rewards/MveiAccuracyReward/std": 0.05260358452796936, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9675000011920929, "rewards/MveiLLMConsistencyReward/std": 0.036222582682967186, "step": 740, "step_time": 70.48028190284967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 224.35, "completions/mean_length": 196.5, "completions/min_length": 166.2, "epoch": 0.5416212286441294, "frac_reward_zero_std": 0.5, "grad_norm": 1.1805001713373837, "kl": 0.049361081700772046, "learning_rate": 4.7297192755414196e-07, "loss": 0.001974627375602722, "reward": 2.4225000143051147, "reward_std": 0.1044437676668167, "rewards/MveiAccuracyReward/mean": 0.9625, "rewards/MveiAccuracyReward/std": 0.0667006328701973, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9599999964237214, "rewards/MveiLLMConsistencyReward/std": 0.07520394884049893, "step": 745, "step_time": 61.17317264787853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 223.45, "completions/mean_length": 195.6, "completions/min_length": 172.1, "epoch": 0.5452562704471101, "frac_reward_zero_std": 0.55, "grad_norm": 0.6491740465797488, "kl": 0.056783065758645536, "learning_rate": 4.669736426893644e-07, "loss": 0.002271723747253418, "reward": 2.352499985694885, "reward_std": 0.0707980003207922, "rewards/MveiAccuracyReward/mean": 0.8875, "rewards/MveiAccuracyReward/std": 0.023145502805709837, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9650000035762787, "rewards/MveiLLMConsistencyReward/std": 0.051695066317915914, "step": 750, "step_time": 58.426505132764575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.5, "completions/mean_length": 202.5125, "completions/min_length": 175.3, "epoch": 0.5488913122500909, "frac_reward_zero_std": 0.3, "grad_norm": 1.0801325006021063, "kl": 0.05887562576681375, "learning_rate": 4.609801281062909e-07, "loss": 0.0023553848266601564, "reward": 2.3574999988079073, "reward_std": 0.15311881639063357, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.09616263210773468, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9449999988079071, "rewards/MveiLLMConsistencyReward/std": 0.07789404280483722, "step": 755, "step_time": 59.36040690355003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 232.05, "completions/mean_length": 202.99375, "completions/min_length": 175.95, "epoch": 0.5525263540530716, "frac_reward_zero_std": 0.45, "grad_norm": 1.110540886692785, "kl": 0.050079621654003856, "learning_rate": 4.5499224949993894e-07, "loss": 0.002003321796655655, "reward": 2.368750011920929, "reward_std": 0.07481234222650528, "rewards/MveiAccuracyReward/mean": 0.93125, "rewards/MveiAccuracyReward/std": 0.025877460837364197, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9375, "rewards/MveiLLMConsistencyReward/std": 0.050710825994610784, "step": 760, "step_time": 60.89374854378402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 223.1, "completions/mean_length": 194.1375, "completions/min_length": 167.9, "epoch": 0.5561613958560524, "frac_reward_zero_std": 0.45, "grad_norm": 1.1723132750119714, "kl": 0.05370624531060457, "learning_rate": 4.490108717512726e-07, "loss": 0.0021477729082107545, "reward": 2.396249973773956, "reward_std": 0.10729667954146863, "rewards/MveiAccuracyReward/mean": 0.925, "rewards/MveiAccuracyReward/std": 0.05850084125995636, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.971250006556511, "rewards/MveiLLMConsistencyReward/std": 0.0538467600941658, "step": 765, "step_time": 57.5007011462003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 238.05, "completions/mean_length": 200.59375, "completions/min_length": 171.35, "epoch": 0.5597964376590331, "frac_reward_zero_std": 0.4, "grad_norm": 0.5824919884796961, "kl": 0.06538908109068871, "learning_rate": 4.4303685880228156e-07, "loss": 0.0026160240173339843, "reward": 2.414999985694885, "reward_std": 0.05796761065721512, "rewards/MveiAccuracyReward/mean": 0.95, "rewards/MveiAccuracyReward/std": 0.0, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9650000035762787, "rewards/MveiLLMConsistencyReward/std": 0.05796760767698288, "step": 770, "step_time": 64.03310044296086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 236.05, "completions/mean_length": 201.54375, "completions/min_length": 173.15, "epoch": 0.5634314794620138, "frac_reward_zero_std": 0.45, "grad_norm": 0.5927449477272995, "kl": 0.06196275260299444, "learning_rate": 4.370710735311941e-07, "loss": 0.002478864789009094, "reward": 2.2337499976158144, "reward_std": 0.1062652364373207, "rewards/MveiAccuracyReward/mean": 0.7875, "rewards/MveiAccuracyReward/std": 0.0667006328701973, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9462500035762786, "rewards/MveiLLMConsistencyReward/std": 0.05085046738386154, "step": 775, "step_time": 67.91525550037622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.75, "completions/mean_length": 201.84375, "completions/min_length": 172.1, "epoch": 0.5670665212649946, "frac_reward_zero_std": 0.35, "grad_norm": 1.7612246576704227, "kl": 0.054537341371178626, "learning_rate": 4.3111437762784305e-07, "loss": 0.002182258665561676, "reward": 2.321249985694885, "reward_std": 0.11211095787584782, "rewards/MveiAccuracyReward/mean": 0.86875, "rewards/MveiAccuracyReward/std": 0.044403792917728425, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9525000005960464, "rewards/MveiLLMConsistencyReward/std": 0.07240946032106876, "step": 780, "step_time": 62.06473238617182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.0, "completions/mean_length": 212.86875, "completions/min_length": 181.3, "epoch": 0.5707015630679753, "frac_reward_zero_std": 0.3, "grad_norm": 1.0629382148331044, "kl": 0.05230696480721235, "learning_rate": 4.2516763146920646e-07, "loss": 0.0020924389362335207, "reward": 2.356249988079071, "reward_std": 0.17326271906495094, "rewards/MveiAccuracyReward/mean": 0.9125, "rewards/MveiAccuracyReward/std": 0.10205597132444381, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.94375, "rewards/MveiLLMConsistencyReward/std": 0.08507692925632, "step": 785, "step_time": 63.057847008854154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 248.8, "completions/mean_length": 199.65625, "completions/min_length": 170.55, "epoch": 0.5743366048709561, "frac_reward_zero_std": 0.55, "grad_norm": 1.1475089053384309, "kl": 0.04890742050483823, "learning_rate": 4.1923169399513425e-07, "loss": 0.001956340670585632, "reward": 2.4231250047683717, "reward_std": 0.09638397991657258, "rewards/MveiAccuracyReward/mean": 0.9625, "rewards/MveiAccuracyReward/std": 0.043555130064487454, "rewards/MveiFormatReward/mean": 0.99375, "rewards/MveiFormatReward/std": 0.01767766922712326, "rewards/MveiLLMConsistencyReward/mean": 0.9637499988079071, "rewards/MveiLLMConsistencyReward/std": 0.05671881660819054, "step": 790, "step_time": 94.11746585927904 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 244.9, "completions/mean_length": 203.5875, "completions/min_length": 173.15, "epoch": 0.5779716466739367, "frac_reward_zero_std": 0.45, "grad_norm": 1.0134052005819736, "kl": 0.056848237104713914, "learning_rate": 4.133074225842851e-07, "loss": 0.0022740095853805544, "reward": 2.3612500011920927, "reward_std": 0.12292048744857312, "rewards/MveiAccuracyReward/mean": 0.925, "rewards/MveiAccuracyReward/std": 0.043555130064487454, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9362500011920929, "rewards/MveiLLMConsistencyReward/std": 0.09131217785179616, "step": 795, "step_time": 65.19886676594615 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 231.8, "completions/mean_length": 200.49375, "completions/min_length": 175.05, "epoch": 0.5816066884769174, "frac_reward_zero_std": 0.35, "grad_norm": 0.9991836210947987, "kl": 0.055218469258397816, "learning_rate": 4.0739567293028764e-07, "loss": 0.002208811044692993, "reward": 2.383749985694885, "reward_std": 0.10879868492484093, "rewards/MveiAccuracyReward/mean": 0.925, "rewards/MveiAccuracyReward/std": 0.043555130064487454, "rewards/MveiFormatReward/mean": 1.0, "rewards/MveiFormatReward/std": 0.0, "rewards/MveiLLMConsistencyReward/mean": 0.9587500035762787, "rewards/MveiLLMConsistencyReward/std": 0.0751047309488058, "step": 800, "step_time": 59.56771714054048 } ], "logging_steps": 5, "max_steps": 1376, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 200, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }