{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.11428571428571428, "eval_steps": 500, "global_step": 100, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2700.4271850585938, "cov_mean": -6.0587970438064076e-05, "cov_std": 0.35307812318205833, "entropy": 0.36767578125, "epoch": 0.001142857142857143, "grad_norm": 0.4553219676017761, "kl": 0.0, "learning_rate": 1e-07, "loss": -0.0382, "reward": 0.7604166893288493, "reward_std": 0.4268697127699852, "rewards/accuracy_reward": 0.25000001303851604, "rewards/format_reward": 0.5104166669771075, "step": 1, "w_high_ratio": 0.2200421690940857, "w_low_ratio": 0.03663695091381669, "w_max": 2.1593789756298065, "w_mean": 1.4711343348026276, "w_min": 6.525355682266089e-35, "w_std": 0.2659660503268242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3127.3958435058594, "cov_mean": -2.155053698515985e-05, "cov_std": 0.310540571808815, "entropy": 0.353515625, "epoch": 0.002285714285714286, "grad_norm": 0.5147930979728699, "kl": 0.0, "learning_rate": 2e-07, "loss": 0.0049, "reward": 0.6458333637565374, "reward_std": 0.4249730706214905, "rewards/accuracy_reward": 0.2812500102445483, "rewards/format_reward": 0.3645833386108279, "step": 2, "w_high_ratio": 0.05183619633316994, "w_low_ratio": 0.036958135198801756, "w_max": 1.8325217366218567, "w_mean": 1.2113382518291473, "w_min": 0.0, "w_std": 0.20957503467798233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3685.479248046875, "cov_mean": -4.653575570046087e-05, "cov_std": 0.29687076807022095, "entropy": 0.45556640625, "epoch": 0.0034285714285714284, "grad_norm": 0.23412799835205078, "kl": 4.607439041137695e-05, "learning_rate": 3e-07, "loss": 0.0316, "reward": 0.2291666753590107, "reward_std": 0.35217108577489853, "rewards/accuracy_reward": 0.05208333395421505, "rewards/format_reward": 0.1770833395421505, "step": 3, "w_high_ratio": 0.0, "w_low_ratio": 0.03702388517558575, "w_max": 1.4749425947666168, "w_mean": 1.0885904878377914, "w_min": 3.5302726539688593e-39, "w_std": 0.1792612113058567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2459.854248046875, "cov_mean": 2.6621969936968526e-05, "cov_std": 0.42025458812713623, "entropy": 0.4140625, "epoch": 0.004571428571428572, "grad_norm": 0.42652714252471924, "kl": 3.8504600524902344e-05, "learning_rate": 4e-07, "loss": 0.0096, "reward": 0.7604167014360428, "reward_std": 0.4487093985080719, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.6354166865348816, "step": 4, "w_high_ratio": 0.2003328688442707, "w_low_ratio": 0.04321334883570671, "w_max": 2.0572283267974854, "w_mean": 1.4974508583545685, "w_min": 0.0, "w_std": 0.26409733667969704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3507.5833740234375, "cov_mean": 7.01061562722316e-05, "cov_std": 0.5858492627739906, "entropy": 0.45458984375, "epoch": 0.005714285714285714, "grad_norm": 0.5644172430038452, "kl": 4.00543212890625e-05, "learning_rate": 5e-07, "loss": 0.0229, "reward": 0.4583333507180214, "reward_std": 0.6347778141498566, "rewards/accuracy_reward": 0.09375000093132257, "rewards/format_reward": 0.3645833507180214, "step": 5, "w_high_ratio": 0.04538528248667717, "w_low_ratio": 0.06710020452737808, "w_max": 1.6610480844974518, "w_mean": 1.1760589182376862, "w_min": 5.419854268836891e-39, "w_std": 0.322455570101738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3396.0104370117188, "cov_mean": 1.043153679347597e-05, "cov_std": 0.3946683406829834, "entropy": 0.4580078125, "epoch": 0.006857142857142857, "grad_norm": 0.47143253684043884, "kl": 4.100799560546875e-05, "learning_rate": 6e-07, "loss": 0.0699, "reward": 0.3125000176951289, "reward_std": 0.33619000762701035, "rewards/accuracy_reward": 0.031250000931322575, "rewards/format_reward": 0.2812500102445483, "step": 6, "w_high_ratio": 0.044003941118717194, "w_low_ratio": 0.04949501994997263, "w_max": 1.6353994011878967, "w_mean": 1.1387673616409302, "w_min": 0.0, "w_std": 0.24893567338585854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3345.291748046875, "cov_mean": -7.031985660432838e-06, "cov_std": 0.4864897131919861, "entropy": 0.38818359375, "epoch": 0.008, "grad_norm": 0.2716952860355377, "kl": 2.446770668029785e-05, "learning_rate": 7e-07, "loss": -0.0398, "reward": 0.7395833507180214, "reward_std": 0.5676274970173836, "rewards/accuracy_reward": 0.1979166679084301, "rewards/format_reward": 0.5416666828095913, "step": 7, "w_high_ratio": 0.0, "w_low_ratio": 0.051307737827301025, "w_max": 1.4717772901058197, "w_mean": 1.1606104969978333, "w_min": 0.0, "w_std": 0.2527086026966572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2934.9583740234375, "cov_mean": -6.692781494166411e-05, "cov_std": 0.31714635342359543, "entropy": 0.36328125, "epoch": 0.009142857142857144, "grad_norm": 0.24377422034740448, "kl": 1.4781951904296875e-05, "learning_rate": 8e-07, "loss": -0.0011, "reward": 0.8750000409781933, "reward_std": 0.5431903004646301, "rewards/accuracy_reward": 0.38541667722165585, "rewards/format_reward": 0.48958336375653744, "step": 8, "w_high_ratio": 0.0, "w_low_ratio": 0.03932169498875737, "w_max": 1.5945640802383423, "w_mean": 1.1880076229572296, "w_min": 0.0, "w_std": 0.20366584323346615 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3466.6250610351562, "cov_mean": -5.504875753103988e-05, "cov_std": 0.3686807006597519, "entropy": 0.4501953125, "epoch": 0.010285714285714285, "grad_norm": 0.3517554700374603, "kl": 4.1872262954711914e-05, "learning_rate": 9e-07, "loss": -0.0325, "reward": 0.4375000074505806, "reward_std": 0.5021138191223145, "rewards/accuracy_reward": 0.13541666883975267, "rewards/format_reward": 0.3020833432674408, "step": 9, "w_high_ratio": 0.021586479619145393, "w_low_ratio": 0.04805116541683674, "w_max": 1.7995203733444214, "w_mean": 1.1711468696594238, "w_min": 1.0441714550433176e-38, "w_std": 0.24575592577457428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2808.8438720703125, "cov_mean": -2.2776248442824e-05, "cov_std": 0.3646387383341789, "entropy": 0.326171875, "epoch": 0.011428571428571429, "grad_norm": 0.44795799255371094, "kl": 2.4825334548950195e-05, "learning_rate": 1e-06, "loss": -0.0625, "reward": 0.6875000223517418, "reward_std": 0.48652123659849167, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.4791666828095913, "step": 10, "w_high_ratio": 0.1090739294886589, "w_low_ratio": 0.04266490042209625, "w_max": 2.1747676730155945, "w_mean": 1.3257513344287872, "w_min": 0.0, "w_std": 0.2564368508756161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3580.1355590820312, "cov_mean": -4.214262844470795e-06, "cov_std": 0.35131024569272995, "entropy": 0.3720703125, "epoch": 0.012571428571428572, "grad_norm": 0.480006605386734, "kl": 3.1560659408569336e-05, "learning_rate": 9.997258721585931e-07, "loss": 0.0246, "reward": 0.3333333507180214, "reward_std": 0.568866454064846, "rewards/accuracy_reward": 0.12500000279396772, "rewards/format_reward": 0.20833334140479565, "step": 11, "w_high_ratio": 0.04230208322405815, "w_low_ratio": 0.05235781241208315, "w_max": 1.6345463395118713, "w_mean": 1.1268693506717682, "w_min": 0.0, "w_std": 0.24075055867433548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2692.9688110351562, "cov_mean": 3.166104261254077e-05, "cov_std": 0.33987831324338913, "entropy": 0.40576171875, "epoch": 0.013714285714285714, "grad_norm": 0.5367066860198975, "kl": 4.026293754577637e-05, "learning_rate": 9.989038226169207e-07, "loss": 0.0037, "reward": 0.7500000149011612, "reward_std": 0.4257746562361717, "rewards/accuracy_reward": 0.16666667442768812, "rewards/format_reward": 0.5833333469927311, "step": 12, "w_high_ratio": 0.20933324098587036, "w_low_ratio": 0.042826405726373196, "w_max": 2.3706648349761963, "w_mean": 1.5066065192222595, "w_min": 0.0, "w_std": 0.3023643344640732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3225.3125610351562, "cov_mean": -1.1440834896347951e-05, "cov_std": 0.25972653180360794, "entropy": 0.38330078125, "epoch": 0.014857142857142857, "grad_norm": 0.5191377997398376, "kl": 3.4749507904052734e-05, "learning_rate": 9.975348529157229e-07, "loss": -0.0342, "reward": 0.6354166865348816, "reward_std": 0.3692361190915108, "rewards/accuracy_reward": 0.22916667722165585, "rewards/format_reward": 0.4062500074505806, "step": 13, "w_high_ratio": 0.059321983717381954, "w_low_ratio": 0.02801903593353927, "w_max": 1.86709925532341, "w_mean": 1.2194385528564453, "w_min": 0.0, "w_std": 0.18418971449136734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3069.2083740234375, "cov_mean": -5.5041197356331395e-05, "cov_std": 0.4479314386844635, "entropy": 0.3857421875, "epoch": 0.016, "grad_norm": 0.4110907018184662, "kl": 3.4496188163757324e-05, "learning_rate": 9.956206309337066e-07, "loss": 0.0262, "reward": 0.6354166716337204, "reward_std": 0.5461946055293083, "rewards/accuracy_reward": 0.20833333861082792, "rewards/format_reward": 0.4270833432674408, "step": 14, "w_high_ratio": 0.05299236997961998, "w_low_ratio": 0.05393570475280285, "w_max": 1.8086846470832825, "w_mean": 1.2297336757183075, "w_min": 0.0, "w_std": 0.29025064408779144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2969.4063110351562, "cov_mean": 4.094462474313332e-05, "cov_std": 0.28705432265996933, "entropy": 0.372802734375, "epoch": 0.017142857142857144, "grad_norm": 0.2711990475654602, "kl": 2.568960189819336e-05, "learning_rate": 9.931634888554935e-07, "loss": 0.0207, "reward": 0.6666666865348816, "reward_std": 0.326509490609169, "rewards/accuracy_reward": 0.2604166669771075, "rewards/format_reward": 0.4062500111758709, "step": 15, "w_high_ratio": 0.0, "w_low_ratio": 0.03214749973267317, "w_max": 1.415438860654831, "w_mean": 1.1301319599151611, "w_min": 0.25, "w_std": 0.17486043274402618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3889.2083740234375, "cov_mean": 1.429535222996492e-05, "cov_std": 0.1915050856769085, "entropy": 0.46337890625, "epoch": 0.018285714285714287, "grad_norm": 0.2660626173019409, "kl": 3.978610038757324e-05, "learning_rate": 9.901664203302124e-07, "loss": 0.0406, "reward": 0.1250000037252903, "reward_std": 0.24391646683216095, "rewards/accuracy_reward": 0.052083334885537624, "rewards/format_reward": 0.07291666883975267, "step": 16, "w_high_ratio": 0.0, "w_low_ratio": 0.027141269762068987, "w_max": 1.2118620574474335, "w_mean": 1.0125571489334106, "w_min": 0.25, "w_std": 0.10200810059905052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2681.927215576172, "cov_mean": 5.457106340145401e-05, "cov_std": 0.4843906760215759, "entropy": 0.46728515625, "epoch": 0.019428571428571427, "grad_norm": 0.5056636929512024, "kl": 4.604458808898926e-05, "learning_rate": 9.866330768241983e-07, "loss": 0.0332, "reward": 0.8020833656191826, "reward_std": 0.5575663819909096, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.5520833432674408, "step": 17, "w_high_ratio": 0.2290888950228691, "w_low_ratio": 0.04584049433469772, "w_max": 2.3289138078689575, "w_mean": 1.451726257801056, "w_min": 3.2229864679470793e-44, "w_std": 0.33130457252264023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3172.822998046875, "cov_mean": 5.8477902712184004e-05, "cov_std": 0.3056485205888748, "entropy": 0.36083984375, "epoch": 0.02057142857142857, "grad_norm": 0.5626700520515442, "kl": 2.8699636459350586e-05, "learning_rate": 9.825677631722435e-07, "loss": -0.0633, "reward": 0.5833333507180214, "reward_std": 0.4684683009982109, "rewards/accuracy_reward": 0.15625000279396772, "rewards/format_reward": 0.4270833469927311, "step": 18, "w_high_ratio": 0.05792288854718208, "w_low_ratio": 0.03927373047918081, "w_max": 1.9344473481178284, "w_mean": 1.2674984335899353, "w_min": 0.0, "w_std": 0.20779520832002163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3062.354217529297, "cov_mean": -3.731243577931309e-05, "cov_std": 0.3898714445531368, "entropy": 0.388671875, "epoch": 0.021714285714285714, "grad_norm": 0.4545780420303345, "kl": 3.999471664428711e-05, "learning_rate": 9.779754323328192e-07, "loss": -0.0387, "reward": 0.8229166772216558, "reward_std": 0.5051202178001404, "rewards/accuracy_reward": 0.37500001583248377, "rewards/format_reward": 0.4479166818782687, "step": 19, "w_high_ratio": 0.01639098348096013, "w_low_ratio": 0.042750393971800804, "w_max": 1.7452828288078308, "w_mean": 1.2359632849693298, "w_min": 0.0, "w_std": 0.25578257627785206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2637.7813415527344, "cov_mean": -5.605046135315206e-05, "cov_std": 0.43815504014492035, "entropy": 0.339599609375, "epoch": 0.022857142857142857, "grad_norm": 0.3516737222671509, "kl": 4.100799560546875e-05, "learning_rate": 9.728616793536587e-07, "loss": -0.0559, "reward": 1.0104167014360428, "reward_std": 0.5058020055294037, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.7187500298023224, "step": 20, "w_high_ratio": 0.11945514753460884, "w_low_ratio": 0.04084030445665121, "w_max": 2.0172511637210846, "w_mean": 1.3857647478580475, "w_min": 0.0, "w_std": 0.24064720049500465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2998.5208740234375, "cov_mean": 2.1053672753623687e-05, "cov_std": 0.19154372438788414, "entropy": 0.423828125, "epoch": 0.024, "grad_norm": 0.40126267075538635, "kl": 0.00011563301086425781, "learning_rate": 9.672327345550543e-07, "loss": 0.0037, "reward": 0.5833333432674408, "reward_std": 0.21227191016077995, "rewards/accuracy_reward": 0.16666666977107525, "rewards/format_reward": 0.4166666679084301, "step": 21, "w_high_ratio": 0.17914548516273499, "w_low_ratio": 0.0219349917024374, "w_max": 1.8282522857189178, "w_mean": 1.416891485452652, "w_min": 0.25, "w_std": 0.1492762751877308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1878.3125457763672, "cov_mean": 9.24348714761436e-05, "cov_std": 0.4086475111544132, "entropy": 0.3994140625, "epoch": 0.025142857142857144, "grad_norm": 0.7276642918586731, "kl": 0.00010776519775390625, "learning_rate": 9.610954559391704e-07, "loss": -0.054, "reward": 1.1041667014360428, "reward_std": 0.40466368943452835, "rewards/accuracy_reward": 0.29166667722165585, "rewards/format_reward": 0.8125000298023224, "step": 22, "w_high_ratio": 0.22925379872322083, "w_low_ratio": 0.04057216551154852, "w_max": 2.499152511358261, "w_mean": 1.6791247129440308, "w_min": 0.0, "w_std": 0.29966018348932266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2871.3438720703125, "cov_mean": -0.0001690889857854927, "cov_std": 0.5799647718667984, "entropy": 0.37353515625, "epoch": 0.026285714285714287, "grad_norm": 0.5503926277160645, "kl": 6.878376007080078e-05, "learning_rate": 9.54457320834625e-07, "loss": -0.0364, "reward": 0.6354166865348816, "reward_std": 0.6411799788475037, "rewards/accuracy_reward": 0.16666667070239782, "rewards/format_reward": 0.4687500149011612, "step": 23, "w_high_ratio": 0.326111301779747, "w_low_ratio": 0.05267245415598154, "w_max": 2.298383414745331, "w_mean": 1.572682112455368, "w_min": 0.0, "w_std": 0.3976207822561264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2923.572998046875, "cov_mean": 7.625344460393535e-05, "cov_std": 0.3671033903956413, "entropy": 0.3544921875, "epoch": 0.027428571428571427, "grad_norm": 0.37181225419044495, "kl": 7.140636444091797e-05, "learning_rate": 9.473264167865171e-07, "loss": -0.0703, "reward": 0.8541666865348816, "reward_std": 0.5709189921617508, "rewards/accuracy_reward": 0.3020833432674408, "rewards/format_reward": 0.5520833432674408, "step": 24, "w_high_ratio": 0.09629620611667633, "w_low_ratio": 0.037845601327717304, "w_max": 2.1263908445835114, "w_mean": 1.3701403737068176, "w_min": 0.25, "w_std": 0.23599852621555328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2825.1458740234375, "cov_mean": -5.495093546414864e-05, "cov_std": 0.29340429976582527, "entropy": 0.4248046875, "epoch": 0.02857142857142857, "grad_norm": 0.32156902551651, "kl": 0.00015220046043395996, "learning_rate": 9.397114317029974e-07, "loss": -0.0438, "reward": 0.677083358168602, "reward_std": 0.3831539675593376, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.4687500149011612, "step": 25, "w_high_ratio": 0.16885175555944443, "w_low_ratio": 0.032804112415760756, "w_max": 1.8952468037605286, "w_mean": 1.3390689194202423, "w_min": 0.25, "w_std": 0.22369066812098026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3157.4584350585938, "cov_mean": 3.823744009423535e-06, "cov_std": 0.15900863334536552, "entropy": 0.4033203125, "epoch": 0.029714285714285714, "grad_norm": 0.18318897485733032, "kl": 2.8640031814575195e-05, "learning_rate": 9.316216432703916e-07, "loss": -0.0211, "reward": 0.71875, "reward_std": 0.1498664766550064, "rewards/accuracy_reward": 0.2291666716337204, "rewards/format_reward": 0.4895833358168602, "step": 26, "w_high_ratio": 0.0625, "w_low_ratio": 0.012461496517062187, "w_max": 1.7156271040439606, "w_mean": 1.2929016053676605, "w_min": 0.5, "w_std": 0.07156859897077084 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3333.5729370117188, "cov_mean": 9.062682693183888e-06, "cov_std": 0.33011873811483383, "entropy": 0.4306640625, "epoch": 0.030857142857142857, "grad_norm": 0.22338484227657318, "kl": 7.772445678710938e-05, "learning_rate": 9.230669076497687e-07, "loss": -0.0295, "reward": 0.5833333507180214, "reward_std": 0.4810289070010185, "rewards/accuracy_reward": 0.15625000558793545, "rewards/format_reward": 0.4270833358168602, "step": 27, "w_high_ratio": 0.125, "w_low_ratio": 0.04519441071897745, "w_max": 1.898216873407364, "w_mean": 1.2821258306503296, "w_min": 0.0, "w_std": 0.1903173178434372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2990.6563110351562, "cov_mean": -2.9691624149563722e-05, "cov_std": 0.27876436710357666, "entropy": 0.392578125, "epoch": 0.032, "grad_norm": 0.2658036947250366, "kl": 0.00012731552124023438, "learning_rate": 9.140576474687263e-07, "loss": -0.0147, "reward": 0.8229166716337204, "reward_std": 0.32865703850984573, "rewards/accuracy_reward": 0.3437500074505806, "rewards/format_reward": 0.4791666716337204, "step": 28, "w_high_ratio": 0.03581305593252182, "w_low_ratio": 0.036258784122765064, "w_max": 1.8826820850372314, "w_mean": 1.2603062689304352, "w_min": 0.25, "w_std": 0.2064611092209816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3477.0625610351562, "cov_mean": 3.658079458546126e-05, "cov_std": 0.3980727344751358, "entropy": 0.41162109375, "epoch": 0.03314285714285714, "grad_norm": 0.5845446586608887, "kl": 0.00014966726303100586, "learning_rate": 9.046048391230247e-07, "loss": -0.03, "reward": 0.4479166781529784, "reward_std": 0.4323580041527748, "rewards/accuracy_reward": 0.13541667256504297, "rewards/format_reward": 0.31250000838190317, "step": 29, "w_high_ratio": 0.03906010463833809, "w_low_ratio": 0.04348241724073887, "w_max": 1.5083820223808289, "w_mean": 1.1707176566123962, "w_min": 0.0, "w_std": 0.23769092187285423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3228.4896240234375, "cov_mean": 8.98454500202206e-05, "cov_std": 0.4796721711754799, "entropy": 0.388671875, "epoch": 0.03428571428571429, "grad_norm": 0.5218952894210815, "kl": 0.0005021095275878906, "learning_rate": 8.9471999940354e-07, "loss": -0.0373, "reward": 0.8229167014360428, "reward_std": 0.6668380200862885, "rewards/accuracy_reward": 0.3020833395421505, "rewards/format_reward": 0.5208333469927311, "step": 30, "w_high_ratio": 0.09154843725264072, "w_low_ratio": 0.050314661115407944, "w_max": 1.7150111198425293, "w_mean": 1.273286134004593, "w_min": 0.0, "w_std": 0.2975125089287758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3316.479248046875, "cov_mean": -2.0371426671772497e-05, "cov_std": 0.32399024814367294, "entropy": 0.39453125, "epoch": 0.03542857142857143, "grad_norm": 0.26446813344955444, "kl": 0.00034368038177490234, "learning_rate": 8.844151714648274e-07, "loss": 0.0163, "reward": 0.4791666828095913, "reward_std": 0.45351940393447876, "rewards/accuracy_reward": 0.17708333767950535, "rewards/format_reward": 0.30208333395421505, "step": 31, "w_high_ratio": 0.16790161281824112, "w_low_ratio": 0.038946577813476324, "w_max": 1.8747785687446594, "w_mean": 1.287878304719925, "w_min": 0.25, "w_std": 0.18489549681544304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3449.291748046875, "cov_mean": -4.376452466203773e-05, "cov_std": 0.3894844390451908, "entropy": 0.443359375, "epoch": 0.036571428571428574, "grad_norm": 0.3080126643180847, "kl": 0.0014617443084716797, "learning_rate": 8.737029101523929e-07, "loss": -0.0432, "reward": 0.7083333469927311, "reward_std": 0.5129974186420441, "rewards/accuracy_reward": 0.30208333395421505, "rewards/format_reward": 0.4062500037252903, "step": 32, "w_high_ratio": 0.0, "w_low_ratio": 0.04653155663982034, "w_max": 1.6309894919395447, "w_mean": 1.170417159795761, "w_min": 0.0, "w_std": 0.2305326908826828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3521.125, "cov_mean": -3.824161467491649e-06, "cov_std": 0.3757542073726654, "entropy": 0.380859375, "epoch": 0.037714285714285714, "grad_norm": 0.2623606026172638, "kl": 0.0004220008850097656, "learning_rate": 8.625962667065487e-07, "loss": -0.0075, "reward": 0.5520833507180214, "reward_std": 0.5886104479432106, "rewards/accuracy_reward": 0.1979166679084301, "rewards/format_reward": 0.3541666828095913, "step": 33, "w_high_ratio": 0.0, "w_low_ratio": 0.04900518897920847, "w_max": 1.5290310084819794, "w_mean": 1.145493358373642, "w_min": 0.0, "w_std": 0.23011762276291847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2698.3020935058594, "cov_mean": 5.745048838434741e-06, "cov_std": 0.4045129865407944, "entropy": 0.4296875, "epoch": 0.038857142857142854, "grad_norm": 0.4748375713825226, "kl": 0.0005691051483154297, "learning_rate": 8.511087728614862e-07, "loss": -0.0352, "reward": 0.979166679084301, "reward_std": 0.4912763461470604, "rewards/accuracy_reward": 0.4270833544433117, "rewards/format_reward": 0.5520833386108279, "step": 34, "w_high_ratio": 0.1756632700562477, "w_low_ratio": 0.04408560786396265, "w_max": 2.0611276030540466, "w_mean": 1.413420170545578, "w_min": 6.104692548832184e-37, "w_std": 0.29224658384919167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3220.1146240234375, "cov_mean": 2.1198811737122014e-06, "cov_std": 0.43479467928409576, "entropy": 0.41845703125, "epoch": 0.04, "grad_norm": 0.34247061610221863, "kl": 0.0007988214492797852, "learning_rate": 8.392544243589427e-07, "loss": 0.0317, "reward": 0.677083358168602, "reward_std": 0.6704623848199844, "rewards/accuracy_reward": 0.2604166753590107, "rewards/format_reward": 0.4166666716337204, "step": 35, "w_high_ratio": 0.0, "w_low_ratio": 0.059821099042892456, "w_max": 1.4533329010009766, "w_mean": 1.1137926578521729, "w_min": 0.0, "w_std": 0.26089195907115936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3597.8333740234375, "cov_mean": 4.828646342502907e-05, "cov_std": 0.35842880606651306, "entropy": 0.49462890625, "epoch": 0.04114285714285714, "grad_norm": 0.41333624720573425, "kl": 0.00125160813331604, "learning_rate": 8.270476638965461e-07, "loss": 0.0601, "reward": 0.2604166744276881, "reward_std": 0.3833026885986328, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.21875000093132257, "step": 36, "w_high_ratio": 0.03994074463844299, "w_low_ratio": 0.04655152838677168, "w_max": 1.508450299501419, "w_mean": 1.11627396941185, "w_min": 0.0, "w_std": 0.19725885801017284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3524.0625610351562, "cov_mean": -4.1810366383288056e-05, "cov_std": 0.08558216691017151, "entropy": 0.4130859375, "epoch": 0.04228571428571429, "grad_norm": 0.5261622071266174, "kl": 0.0005668401718139648, "learning_rate": 8.145033635316128e-07, "loss": -0.0234, "reward": 0.2291666716337204, "reward_std": 0.07216878235340118, "rewards/accuracy_reward": 0.010416666977107525, "rewards/format_reward": 0.21875, "step": 37, "w_high_ratio": 0.08015145361423492, "w_low_ratio": 0.006487013306468725, "w_max": 1.6550433039665222, "w_mean": 1.1928074657917023, "w_min": 0.75, "w_std": 0.07643502950668335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3520.6146240234375, "cov_mean": 2.719235999393277e-05, "cov_std": 0.1502552330493927, "entropy": 0.44580078125, "epoch": 0.04342857142857143, "grad_norm": 0.23012137413024902, "kl": 0.00047528743743896484, "learning_rate": 8.01636806561836e-07, "loss": -0.0058, "reward": 0.364583358168602, "reward_std": 0.22543279081583023, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.1979166716337204, "step": 38, "w_high_ratio": 0.0, "w_low_ratio": 0.017723015509545803, "w_max": 1.3341983556747437, "w_mean": 1.0956220924854279, "w_min": 0.5, "w_std": 0.08639993146061897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3074.104248046875, "cov_mean": 3.08786220557522e-05, "cov_std": 0.2986413836479187, "entropy": 0.35693359375, "epoch": 0.044571428571428574, "grad_norm": 0.19847992062568665, "kl": 0.0011906623840332031, "learning_rate": 7.884636689049422e-07, "loss": -0.0017, "reward": 0.7187500074505806, "reward_std": 0.24960162490606308, "rewards/accuracy_reward": 0.2604166669771075, "rewards/format_reward": 0.4583333432674408, "step": 39, "w_high_ratio": 0.0, "w_low_ratio": 0.030357226729393005, "w_max": 1.4879463911056519, "w_mean": 1.1616290509700775, "w_min": 0.5, "w_std": 0.15971330553293228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2789.6875610351562, "cov_mean": -5.7511609156790655e-05, "cov_std": 0.31858188658952713, "entropy": 0.38525390625, "epoch": 0.045714285714285714, "grad_norm": 0.2732243835926056, "kl": 0.00257110595703125, "learning_rate": 7.75e-07, "loss": 0.0166, "reward": 0.7500000298023224, "reward_std": 0.41665229201316833, "rewards/accuracy_reward": 0.20833334047347307, "rewards/format_reward": 0.5416666753590107, "step": 40, "w_high_ratio": 0.04059065505862236, "w_low_ratio": 0.037114882841706276, "w_max": 1.8804034292697906, "w_mean": 1.3260886669158936, "w_min": 0.0, "w_std": 0.2053362838923931 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3289.0209350585938, "cov_mean": 1.5465679098269902e-05, "cov_std": 0.45656104385852814, "entropy": 0.37890625, "epoch": 0.046857142857142854, "grad_norm": 0.26020023226737976, "kl": 0.0005598068237304688, "learning_rate": 7.612622032536507e-07, "loss": -0.0115, "reward": 0.489583358168602, "reward_std": 0.507674403488636, "rewards/accuracy_reward": 0.1041666716337204, "rewards/format_reward": 0.3854166865348816, "step": 41, "w_high_ratio": 0.1044064536690712, "w_low_ratio": 0.047306024469435215, "w_max": 2.001980274915695, "w_mean": 1.2991511821746826, "w_min": 3.6180180170197286e-34, "w_std": 0.2728148624300957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3092.7709350585938, "cov_mean": 4.073379568581004e-05, "cov_std": 0.20339186489582062, "entropy": 0.4619140625, "epoch": 0.048, "grad_norm": 0.27600958943367004, "kl": 0.0006108283996582031, "learning_rate": 7.472670160550848e-07, "loss": 0.0105, "reward": 0.3125000037252903, "reward_std": 0.18993012979626656, "rewards/accuracy_reward": 0.02083333395421505, "rewards/format_reward": 0.29166666977107525, "step": 42, "w_high_ratio": 0.05672482028603554, "w_low_ratio": 0.01900631980970502, "w_max": 1.7394262850284576, "w_mean": 1.195200800895691, "w_min": 0.25, "w_std": 0.11057326570153236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3247.92724609375, "cov_mean": 4.6732177906960715e-05, "cov_std": 0.2612141128629446, "entropy": 0.40087890625, "epoch": 0.04914285714285714, "grad_norm": 0.28032609820365906, "kl": 0.001550912857055664, "learning_rate": 7.330314893841101e-07, "loss": -0.0352, "reward": 0.4166666818782687, "reward_std": 0.2987091466784477, "rewards/accuracy_reward": 0.13541666977107525, "rewards/format_reward": 0.2812500027939677, "step": 43, "w_high_ratio": 0.041576627641916275, "w_low_ratio": 0.03293204982765019, "w_max": 1.584185391664505, "w_mean": 1.1372461915016174, "w_min": 0.0, "w_std": 0.16969743371009827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2958.885482788086, "cov_mean": -4.934846765536349e-05, "cov_std": 0.305513933300972, "entropy": 0.41455078125, "epoch": 0.05028571428571429, "grad_norm": 0.5735454559326172, "kl": 0.001309037208557129, "learning_rate": 7.185729670371604e-07, "loss": -0.0235, "reward": 0.6770833432674408, "reward_std": 0.37915313243865967, "rewards/accuracy_reward": 0.2291666679084301, "rewards/format_reward": 0.4479166716337204, "step": 44, "w_high_ratio": 0.05263275280594826, "w_low_ratio": 0.033476518001407385, "w_max": 1.7282162308692932, "w_mean": 1.2310035228729248, "w_min": 0.25, "w_std": 0.20509596168994904 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3681.7500610351562, "cov_mean": 5.253156632534228e-05, "cov_std": 0.27400747686624527, "entropy": 0.4130859375, "epoch": 0.05142857142857143, "grad_norm": 0.22661462426185608, "kl": 0.001455068588256836, "learning_rate": 7.039090644965509e-07, "loss": 0.0004, "reward": 0.3854166716337204, "reward_std": 0.4831271469593048, "rewards/accuracy_reward": 0.16666666977107525, "rewards/format_reward": 0.2187500111758709, "step": 45, "w_high_ratio": 0.0, "w_low_ratio": 0.036470845341682434, "w_max": 1.3770877122879028, "w_mean": 1.0782505869865417, "w_min": 0.25, "w_std": 0.16539720818400383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3533.5209350585938, "cov_mean": 2.201543838964426e-05, "cov_std": 0.20324882864952087, "entropy": 0.4873046875, "epoch": 0.052571428571428575, "grad_norm": 0.42943075299263, "kl": 0.0011663436889648438, "learning_rate": 6.890576474687263e-07, "loss": 0.0747, "reward": 0.18750000558793545, "reward_std": 0.185950867831707, "rewards/accuracy_reward": 0.02083333395421505, "rewards/format_reward": 0.1666666679084301, "step": 46, "w_high_ratio": 0.10795218497514725, "w_low_ratio": 0.02661217050626874, "w_max": 1.561707764863968, "w_mean": 1.1653469800949097, "w_min": 0.0, "w_std": 0.1572987660765648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3152.2188720703125, "cov_mean": -8.783125122135971e-05, "cov_std": 0.4741008821874857, "entropy": 0.388671875, "epoch": 0.053714285714285714, "grad_norm": 0.45350003242492676, "kl": 0.000950932502746582, "learning_rate": 6.740368101176495e-07, "loss": -0.0528, "reward": 0.8125000400468707, "reward_std": 0.599571980535984, "rewards/accuracy_reward": 0.3125000111758709, "rewards/format_reward": 0.5000000102445483, "step": 47, "w_high_ratio": 0.09517703950405121, "w_low_ratio": 0.0510673513635993, "w_max": 1.8478549420833588, "w_mean": 1.2849175333976746, "w_min": 0.0, "w_std": 0.2784101627767086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2912.8334350585938, "cov_mean": 4.9970367399509996e-08, "cov_std": 0.39979860931634903, "entropy": 0.39013671875, "epoch": 0.054857142857142854, "grad_norm": 0.42112770676612854, "kl": 0.00843667984008789, "learning_rate": 6.588648530198504e-07, "loss": -0.012, "reward": 0.7083333469927311, "reward_std": 0.5211797282099724, "rewards/accuracy_reward": 0.25000001303851604, "rewards/format_reward": 0.45833334140479565, "step": 48, "w_high_ratio": 0.1536145769059658, "w_low_ratio": 0.049599490128457546, "w_max": 2.127849966287613, "w_mean": 1.337787538766861, "w_min": 0.0, "w_std": 0.3069620132446289 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2544.666748046875, "cov_mean": -1.3007749203097774e-05, "cov_std": 0.3665553033351898, "entropy": 0.365234375, "epoch": 0.056, "grad_norm": 0.326923668384552, "kl": 0.007767677307128906, "learning_rate": 6.435602608679916e-07, "loss": 0.028, "reward": 0.770833358168602, "reward_std": 0.4556447044014931, "rewards/accuracy_reward": 0.17708333861082792, "rewards/format_reward": 0.5937500149011612, "step": 49, "w_high_ratio": 0.07310386933386326, "w_low_ratio": 0.03729367163032293, "w_max": 1.794796198606491, "w_mean": 1.2659293115139008, "w_min": 6.92190445412429e-34, "w_std": 0.23256080597639084 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3237.2916870117188, "cov_mean": 1.4073829220251355e-05, "cov_std": 0.2659311108291149, "entropy": 0.33642578125, "epoch": 0.05714285714285714, "grad_norm": 0.18902550637722015, "kl": 0.0010194778442382812, "learning_rate": 6.281416799501187e-07, "loss": -0.0195, "reward": 0.645833345130086, "reward_std": 0.36775920540094376, "rewards/accuracy_reward": 0.2916666669771075, "rewards/format_reward": 0.3541666781529784, "step": 50, "w_high_ratio": 0.03271413967013359, "w_low_ratio": 0.030921938829123974, "w_max": 1.3970885872840881, "w_mean": 1.1265143752098083, "w_min": 0.25, "w_std": 0.15340184792876244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2497.2188720703125, "cov_mean": 0.00010049804541267804, "cov_std": 0.31800365447998047, "entropy": 0.451171875, "epoch": 0.05828571428571429, "grad_norm": 0.3681160509586334, "kl": 0.0043849945068359375, "learning_rate": 6.126278954320294e-07, "loss": 0.0301, "reward": 0.739583358168602, "reward_std": 0.32177041098475456, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.5312500149011612, "step": 51, "w_high_ratio": 0.0, "w_low_ratio": 0.038371569477021694, "w_max": 1.5697456300258636, "w_mean": 1.1596841514110565, "w_min": 0.25, "w_std": 0.17389511317014694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3154.15625, "cov_mean": 1.5342720871558413e-08, "cov_std": 0.3799358010292053, "entropy": 0.38623046875, "epoch": 0.05942857142857143, "grad_norm": 0.2101733535528183, "kl": 0.0017533302307128906, "learning_rate": 5.97037808470444e-07, "loss": 0.0285, "reward": 0.6979166697710752, "reward_std": 0.5342019759118557, "rewards/accuracy_reward": 0.2812500074505806, "rewards/format_reward": 0.41666667722165585, "step": 52, "w_high_ratio": 0.0, "w_low_ratio": 0.04498880193568766, "w_max": 1.3936763405799866, "w_mean": 1.130004495382309, "w_min": 0.0, "w_std": 0.19680343009531498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3053.7708740234375, "cov_mean": 1.7406931874575093e-05, "cov_std": 0.46154169738292694, "entropy": 0.421875, "epoch": 0.060571428571428575, "grad_norm": 0.33937692642211914, "kl": 0.001083970069885254, "learning_rate": 5.813904131848564e-07, "loss": -0.0326, "reward": 0.8645833507180214, "reward_std": 0.6376037672162056, "rewards/accuracy_reward": 0.3020833358168602, "rewards/format_reward": 0.5625000074505806, "step": 53, "w_high_ratio": 0.04103509336709976, "w_low_ratio": 0.05415187170729041, "w_max": 1.5957177877426147, "w_mean": 1.1700108051300049, "w_min": 0.0, "w_std": 0.24823076836764812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2899.4688720703125, "cov_mean": -8.303408776555443e-06, "cov_std": 0.4274276793003082, "entropy": 0.37158203125, "epoch": 0.061714285714285715, "grad_norm": 0.4694461524486542, "kl": 0.0010991096496582031, "learning_rate": 5.657047735161255e-07, "loss": -0.0728, "reward": 1.020833358168602, "reward_std": 0.6003277003765106, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.5625000223517418, "step": 54, "w_high_ratio": 0.10438984632492065, "w_low_ratio": 0.044082850217819214, "w_max": 1.875702828168869, "w_mean": 1.3796275556087494, "w_min": 0.0, "w_std": 0.28376903757452965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3251.5626220703125, "cov_mean": 5.693810294360446e-05, "cov_std": 0.32220450788736343, "entropy": 0.39306640625, "epoch": 0.06285714285714286, "grad_norm": 0.3240751326084137, "kl": 0.0013048648834228516, "learning_rate": 5.5e-07, "loss": -0.0123, "reward": 0.5625000149011612, "reward_std": 0.4148038476705551, "rewards/accuracy_reward": 0.20833333674818277, "rewards/format_reward": 0.3541666753590107, "step": 55, "w_high_ratio": 0.06664891541004181, "w_low_ratio": 0.036852192133665085, "w_max": 1.7040621638298035, "w_mean": 1.2043538689613342, "w_min": 0.25, "w_std": 0.2000352405011654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3137.3750915527344, "cov_mean": 6.871268851682544e-07, "cov_std": 0.27662914246320724, "entropy": 0.38330078125, "epoch": 0.064, "grad_norm": 0.2951548397541046, "kl": 0.0005192756652832031, "learning_rate": 5.342952264838747e-07, "loss": -0.0268, "reward": 0.6770833507180214, "reward_std": 0.280517403036356, "rewards/accuracy_reward": 0.21875, "rewards/format_reward": 0.4583333507180214, "step": 56, "w_high_ratio": 0.0, "w_low_ratio": 0.030007817316800356, "w_max": 1.4776403903961182, "w_mean": 1.1647382080554962, "w_min": 1.7701902850583252e-42, "w_std": 0.1563316509127617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3544.9896240234375, "cov_mean": 2.435890860397194e-05, "cov_std": 0.3566681556403637, "entropy": 0.31005859375, "epoch": 0.06514285714285714, "grad_norm": 0.5244222283363342, "kl": 0.00036072731018066406, "learning_rate": 5.186095868151436e-07, "loss": -0.0007, "reward": 0.5208333386108279, "reward_std": 0.5608532726764679, "rewards/accuracy_reward": 0.16666667442768812, "rewards/format_reward": 0.3541666818782687, "step": 57, "w_high_ratio": 0.0016966089606285095, "w_low_ratio": 0.046746447682380676, "w_max": 1.4659978747367859, "w_mean": 1.1122615337371826, "w_min": 0.0, "w_std": 0.21731951087713242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2408.010467529297, "cov_mean": 6.246490556804929e-06, "cov_std": 0.37900903820991516, "entropy": 0.333251953125, "epoch": 0.06628571428571428, "grad_norm": 0.278490275144577, "kl": 0.0038089752197265625, "learning_rate": 5.02962191529556e-07, "loss": 0.0026, "reward": 0.9479167088866234, "reward_std": 0.44713668525218964, "rewards/accuracy_reward": 0.291666672565043, "rewards/format_reward": 0.6562500111758709, "step": 58, "w_high_ratio": 0.10547220706939697, "w_low_ratio": 0.040943967178463936, "w_max": 1.8222440481185913, "w_mean": 1.3768556416034698, "w_min": 3.915851168125891e-30, "w_std": 0.2507038414478302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3130.7916870117188, "cov_mean": 4.25004100179649e-06, "cov_std": 0.2073173113167286, "entropy": 0.332763671875, "epoch": 0.06742857142857143, "grad_norm": 0.354891836643219, "kl": 0.00041985511779785156, "learning_rate": 4.873721045679706e-07, "loss": 0.0052, "reward": 0.4791666669771075, "reward_std": 0.2661800682544708, "rewards/accuracy_reward": 0.1458333358168602, "rewards/format_reward": 0.3333333386108279, "step": 59, "w_high_ratio": 0.0, "w_low_ratio": 0.027361327316612005, "w_max": 1.3359772562980652, "w_mean": 1.097507894039154, "w_min": 0.25, "w_std": 0.14073412120342255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3174.041748046875, "cov_mean": -5.75426925024658e-05, "cov_std": 0.36878733336925507, "entropy": 0.353515625, "epoch": 0.06857142857142857, "grad_norm": 0.23771578073501587, "kl": 0.0018591880798339844, "learning_rate": 4.7185832004988133e-07, "loss": 0.0192, "reward": 0.5416666865348816, "reward_std": 0.4686477780342102, "rewards/accuracy_reward": 0.14583333861082792, "rewards/format_reward": 0.3958333432674408, "step": 60, "w_high_ratio": 0.04001428559422493, "w_low_ratio": 0.03881241288036108, "w_max": 1.5190648436546326, "w_mean": 1.161254346370697, "w_min": 0.0, "w_std": 0.19259942695498466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3259.229248046875, "cov_mean": 1.2544531273306347e-05, "cov_std": 0.32379359379410744, "entropy": 0.361328125, "epoch": 0.06971428571428571, "grad_norm": 0.3283206522464752, "kl": 0.0008237361907958984, "learning_rate": 4.5643973913200837e-07, "loss": -0.0185, "reward": 0.614583358168602, "reward_std": 0.4510444663465023, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.4479166716337204, "step": 61, "w_high_ratio": 0.0, "w_low_ratio": 0.04093870846554637, "w_max": 1.5261969566345215, "w_mean": 1.1891083717346191, "w_min": 0.25, "w_std": 0.2139909453690052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2809.1771850585938, "cov_mean": 3.0659541152999736e-05, "cov_std": 0.5087622255086899, "entropy": 0.326904296875, "epoch": 0.07085714285714285, "grad_norm": 0.6958292722702026, "kl": 0.022649765014648438, "learning_rate": 4.4113514698014953e-07, "loss": -0.0085, "reward": 0.8958333656191826, "reward_std": 0.6562541723251343, "rewards/accuracy_reward": 0.28125000558793545, "rewards/format_reward": 0.6145833507180214, "step": 62, "w_high_ratio": 0.058254461735486984, "w_low_ratio": 0.053510635159909725, "w_max": 1.9464432299137115, "w_mean": 1.336916297674179, "w_min": 0.0, "w_std": 0.31536465138196945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2338.1250610351562, "cov_mean": -3.693009693961358e-05, "cov_std": 0.37869442999362946, "entropy": 0.3642578125, "epoch": 0.072, "grad_norm": 0.3641963601112366, "kl": 0.0020456314086914062, "learning_rate": 4.2596318988235037e-07, "loss": -0.0179, "reward": 1.1562500596046448, "reward_std": 0.517615370452404, "rewards/accuracy_reward": 0.4062500074505806, "rewards/format_reward": 0.7500000223517418, "step": 63, "w_high_ratio": 0.10371162719093263, "w_low_ratio": 0.03615910932421684, "w_max": 2.5228151381015778, "w_mean": 1.538686066865921, "w_min": 4.8989017643768394e-36, "w_std": 0.2665465883910656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3296.385528564453, "cov_mean": -5.260490161163034e-05, "cov_std": 0.32132588326931, "entropy": 0.416015625, "epoch": 0.07314285714285715, "grad_norm": 0.26374608278274536, "kl": 0.0010199546813964844, "learning_rate": 4.1094235253127374e-07, "loss": -0.0008, "reward": 0.6145833656191826, "reward_std": 0.4967978894710541, "rewards/accuracy_reward": 0.22916667722165585, "rewards/format_reward": 0.3854166716337204, "step": 64, "w_high_ratio": 0.125, "w_low_ratio": 0.039727743715047836, "w_max": 1.6750397682189941, "w_mean": 1.3110667765140533, "w_min": 7.912817135887769e-41, "w_std": 0.19634342193603516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3007.2500610351562, "cov_mean": 3.157142418785952e-05, "cov_std": 0.27999811992049217, "entropy": 0.32763671875, "epoch": 0.07428571428571429, "grad_norm": 0.18390174210071564, "kl": 0.0012006759643554688, "learning_rate": 3.9609093550344907e-07, "loss": 0.0048, "reward": 0.6562500074505806, "reward_std": 0.3042946420609951, "rewards/accuracy_reward": 0.1875, "rewards/format_reward": 0.4687500074505806, "step": 65, "w_high_ratio": 0.0, "w_low_ratio": 0.03495390526950359, "w_max": 1.4882625937461853, "w_mean": 1.1612237989902496, "w_min": 0.0, "w_std": 0.14175301045179367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2299.2396240234375, "cov_mean": 2.2165444875099638e-05, "cov_std": 0.1351991407573223, "entropy": 0.306640625, "epoch": 0.07542857142857143, "grad_norm": 0.42320746183395386, "kl": 0.0011644363403320312, "learning_rate": 3.8142703296283953e-07, "loss": -0.0144, "reward": 0.8541666865348816, "reward_std": 0.1657295972108841, "rewards/accuracy_reward": 0.354166679084301, "rewards/format_reward": 0.5, "step": 66, "w_high_ratio": 0.0625, "w_low_ratio": 0.015947649022564292, "w_max": 1.6368336379528046, "w_mean": 1.2440811097621918, "w_min": 0.25, "w_std": 0.09564665146172047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3708.572998046875, "cov_mean": 9.353170185022464e-06, "cov_std": 0.17532559856772423, "entropy": 0.3408203125, "epoch": 0.07657142857142857, "grad_norm": 0.18465003371238708, "kl": 0.0010061264038085938, "learning_rate": 3.6696851061588994e-07, "loss": 0.0047, "reward": 0.20833333861082792, "reward_std": 0.1927690953016281, "rewards/accuracy_reward": 0.041666666977107525, "rewards/format_reward": 0.1666666716337204, "step": 67, "w_high_ratio": 0.0, "w_low_ratio": 0.021573648788034916, "w_max": 1.3159178793430328, "w_mean": 1.0860951095819473, "w_min": 0.25, "w_std": 0.11260665208101273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2515.729232788086, "cov_mean": 1.1593250746955164e-05, "cov_std": 0.31169629842042923, "entropy": 0.4072265625, "epoch": 0.07771428571428571, "grad_norm": 0.39554792642593384, "kl": 0.009487152099609375, "learning_rate": 3.5273298394491515e-07, "loss": -0.0017, "reward": 0.7812500298023224, "reward_std": 0.47302668541669846, "rewards/accuracy_reward": 0.2187500074505806, "rewards/format_reward": 0.5625000074505806, "step": 68, "w_high_ratio": 0.20789940655231476, "w_low_ratio": 0.03476168913766742, "w_max": 2.3950938880443573, "w_mean": 1.504720538854599, "w_min": 0.25, "w_std": 0.20618349313735962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2761.9896240234375, "cov_mean": -1.7451284293201752e-05, "cov_std": 0.2980487421154976, "entropy": 0.43212890625, "epoch": 0.07885714285714286, "grad_norm": 0.39313724637031555, "kl": 0.003513336181640625, "learning_rate": 3.387377967463493e-07, "loss": -0.0408, "reward": 0.4791666828095913, "reward_std": 0.3629096820950508, "rewards/accuracy_reward": 0.041666666977107525, "rewards/format_reward": 0.4375000260770321, "step": 69, "w_high_ratio": 0.06426303647458553, "w_low_ratio": 0.03391136787831783, "w_max": 1.8445489406585693, "w_mean": 1.2533070147037506, "w_min": 0.0, "w_std": 0.1965021900832653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3302.885498046875, "cov_mean": 2.7586031137616374e-05, "cov_std": 0.25518757849931717, "entropy": 0.35498046875, "epoch": 0.08, "grad_norm": 0.21536409854888916, "kl": 0.0017971992492675781, "learning_rate": 3.250000000000001e-07, "loss": 0.0226, "reward": 0.489583358168602, "reward_std": 0.3120904117822647, "rewards/accuracy_reward": 0.09375000279396772, "rewards/format_reward": 0.3958333432674408, "step": 70, "w_high_ratio": 0.0, "w_low_ratio": 0.035551197826862335, "w_max": 1.2505681812763214, "w_mean": 1.0716167390346527, "w_min": 0.25, "w_std": 0.13985217362642288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2877.8958740234375, "cov_mean": -1.1374457244528458e-05, "cov_std": 0.2578735649585724, "entropy": 0.40185546875, "epoch": 0.08114285714285714, "grad_norm": 0.5054724812507629, "kl": 0.0020956993103027344, "learning_rate": 3.115363310950578e-07, "loss": 0.0119, "reward": 0.6145833637565374, "reward_std": 0.24616123735904694, "rewards/accuracy_reward": 0.2291666716337204, "rewards/format_reward": 0.38541666977107525, "step": 71, "w_high_ratio": 0.11903085745871067, "w_low_ratio": 0.02461471315473318, "w_max": 1.9073027968406677, "w_mean": 1.2823512256145477, "w_min": 0.25, "w_std": 0.20722979307174683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3255.104248046875, "cov_mean": 2.442240383970784e-05, "cov_std": 0.49785537272691727, "entropy": 0.4853515625, "epoch": 0.08228571428571428, "grad_norm": 0.4614424407482147, "kl": 0.004761219024658203, "learning_rate": 2.9836319343816397e-07, "loss": 0.002, "reward": 0.4583333358168602, "reward_std": 0.5298330634832382, "rewards/accuracy_reward": 0.06250000279396772, "rewards/format_reward": 0.3958333432674408, "step": 72, "w_high_ratio": 0.01689874939620495, "w_low_ratio": 0.058014022186398506, "w_max": 1.7353839576244354, "w_mean": 1.2576677203178406, "w_min": 0.0, "w_std": 0.312557902187109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3811.3541870117188, "cov_mean": 9.231509466189891e-06, "cov_std": 0.21682696044445038, "entropy": 0.439453125, "epoch": 0.08342857142857144, "grad_norm": 0.11269888281822205, "kl": 0.0003600120544433594, "learning_rate": 2.854966364683872e-07, "loss": 0.0008, "reward": 0.3437500149011612, "reward_std": 0.34103409945964813, "rewards/accuracy_reward": 0.1354166716337204, "rewards/format_reward": 0.2083333432674408, "step": 73, "w_high_ratio": 0.0, "w_low_ratio": 0.025642000138759613, "w_max": 1.2365370392799377, "w_mean": 1.0363657176494598, "w_min": 0.25, "w_std": 0.10616981983184814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3564.8750610351562, "cov_mean": 7.751962675683899e-05, "cov_std": 0.2809390053153038, "entropy": 0.41064453125, "epoch": 0.08457142857142858, "grad_norm": 0.23569634556770325, "kl": 0.0016646385192871094, "learning_rate": 2.729523361034538e-07, "loss": 0.0156, "reward": 0.479166679084301, "reward_std": 0.4482262283563614, "rewards/accuracy_reward": 0.21875000186264515, "rewards/format_reward": 0.2604166716337204, "step": 74, "w_high_ratio": 0.0, "w_low_ratio": 0.04224945977330208, "w_max": 1.3714891374111176, "w_mean": 1.06686469912529, "w_min": 0.25, "w_std": 0.17370857670903206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3226.3126220703125, "cov_mean": 6.714556593578891e-05, "cov_std": 0.3720974437892437, "entropy": 0.36962890625, "epoch": 0.08571428571428572, "grad_norm": 0.22486244142055511, "kl": 0.0015187263488769531, "learning_rate": 2.6074557564105724e-07, "loss": 0.0805, "reward": 0.6041666939854622, "reward_std": 0.3942909240722656, "rewards/accuracy_reward": 0.1979166716337204, "rewards/format_reward": 0.40625001303851604, "step": 75, "w_high_ratio": 0.05757378041744232, "w_low_ratio": 0.045274180360138416, "w_max": 1.8611223101615906, "w_mean": 1.1981039345264435, "w_min": 0.0, "w_std": 0.2294696494936943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3215.9271850585938, "cov_mean": 1.099553310268675e-05, "cov_std": 0.2080927938222885, "entropy": 0.3857421875, "epoch": 0.08685714285714285, "grad_norm": 0.12048080563545227, "kl": 0.0004360675811767578, "learning_rate": 2.488912271385139e-07, "loss": -0.0073, "reward": 0.4791666865348816, "reward_std": 0.2375478893518448, "rewards/accuracy_reward": 0.0625, "rewards/format_reward": 0.4166666865348816, "step": 76, "w_high_ratio": 0.1249999962747097, "w_low_ratio": 0.027999846264719963, "w_max": 1.7428039014339447, "w_mean": 1.2136583924293518, "w_min": 0.25, "w_std": 0.10646160691976547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3392.4583740234375, "cov_mean": -4.415775742927508e-05, "cov_std": 0.3749671131372452, "entropy": 0.39599609375, "epoch": 0.088, "grad_norm": 0.2782253324985504, "kl": 0.0008301734924316406, "learning_rate": 2.374037332934512e-07, "loss": -0.016, "reward": 0.48958333395421505, "reward_std": 0.39483094960451126, "rewards/accuracy_reward": 0.1354166716337204, "rewards/format_reward": 0.35416667722165585, "step": 77, "w_high_ratio": 0.0396752767264843, "w_low_ratio": 0.03982406808063388, "w_max": 1.5143637657165527, "w_mean": 1.1384893357753754, "w_min": 0.0, "w_std": 0.20439331606030464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3471.21875, "cov_mean": -2.8236715252205613e-05, "cov_std": 0.4440799057483673, "entropy": 0.36767578125, "epoch": 0.08914285714285715, "grad_norm": 0.34334930777549744, "kl": 0.00042557716369628906, "learning_rate": 2.2629708984760706e-07, "loss": 0.0139, "reward": 0.6041666902601719, "reward_std": 0.5486822947859764, "rewards/accuracy_reward": 0.2708333358168602, "rewards/format_reward": 0.33333334513008595, "step": 78, "w_high_ratio": 0.05815339833498001, "w_low_ratio": 0.04555406514555216, "w_max": 1.7224481999874115, "w_mean": 1.199266105890274, "w_min": 0.0, "w_std": 0.2618200555443764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2512.6146697998047, "cov_mean": 4.2871850382653065e-05, "cov_std": 0.2797487545758486, "entropy": 0.332275390625, "epoch": 0.09028571428571429, "grad_norm": 0.3756020963191986, "kl": 0.0013785362243652344, "learning_rate": 2.1558482853517253e-07, "loss": 0.0565, "reward": 0.7604166865348816, "reward_std": 0.3231862857937813, "rewards/accuracy_reward": 0.20833333861082792, "rewards/format_reward": 0.5520833432674408, "step": 79, "w_high_ratio": 0.05819739028811455, "w_low_ratio": 0.030684792436659336, "w_max": 2.2010356187820435, "w_mean": 1.3371951580047607, "w_min": 0.0, "w_std": 0.20669112168252468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3588.604248046875, "cov_mean": -7.164392627601046e-06, "cov_std": 0.2609938234090805, "entropy": 0.46240234375, "epoch": 0.09142857142857143, "grad_norm": 0.20949065685272217, "kl": 0.0011081695556640625, "learning_rate": 2.0528000059645995e-07, "loss": 0.0063, "reward": 0.500000013038516, "reward_std": 0.4404493272304535, "rewards/accuracy_reward": 0.17708333488553762, "rewards/format_reward": 0.3229166781529784, "step": 80, "w_high_ratio": 0.0, "w_low_ratio": 0.03598626330494881, "w_max": 1.3815899193286896, "w_mean": 1.1029056310653687, "w_min": 0.25, "w_std": 0.14113666117191315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3374.7188110351562, "cov_mean": 3.345898390705315e-05, "cov_std": 0.17040981724858284, "entropy": 0.51806640625, "epoch": 0.09257142857142857, "grad_norm": 0.16646912693977356, "kl": 0.0033516883850097656, "learning_rate": 1.9539516087697517e-07, "loss": -0.0106, "reward": 0.3750000102445483, "reward_std": 0.21947956830263138, "rewards/accuracy_reward": 0.09375000558793545, "rewards/format_reward": 0.2812500027939677, "step": 81, "w_high_ratio": 0.0625, "w_low_ratio": 0.021185603458434343, "w_max": 1.6720820665359497, "w_mean": 1.217385858297348, "w_min": 0.25, "w_std": 0.0957618746906519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3105.322998046875, "cov_mean": -3.050990926567465e-05, "cov_std": 0.23843222856521606, "entropy": 0.42138671875, "epoch": 0.09371428571428571, "grad_norm": 0.34804272651672363, "kl": 0.0018186569213867188, "learning_rate": 1.8594235253127372e-07, "loss": -0.0113, "reward": 0.6458333507180214, "reward_std": 0.31302163377404213, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.3958333395421505, "step": 82, "w_high_ratio": 0.0, "w_low_ratio": 0.02489804197102785, "w_max": 1.6160834431648254, "w_mean": 1.1803554594516754, "w_min": 0.0, "w_std": 0.15118960477411747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3147.4583740234375, "cov_mean": -2.670003959792666e-05, "cov_std": 0.2024751603603363, "entropy": 0.43408203125, "epoch": 0.09485714285714286, "grad_norm": 0.24946807324886322, "kl": 0.0009236335754394531, "learning_rate": 1.7693309235023127e-07, "loss": 0.0343, "reward": 0.3854166679084301, "reward_std": 0.24196770787239075, "rewards/accuracy_reward": 0.10416666977107525, "rewards/format_reward": 0.28125000558793545, "step": 83, "w_high_ratio": 0.0, "w_low_ratio": 0.030171910300850868, "w_max": 1.2905269861221313, "w_mean": 1.0636517405509949, "w_min": 0.25, "w_std": 0.1290601659566164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3317.5625, "cov_mean": 4.104694835405098e-05, "cov_std": 0.4057937487959862, "entropy": 0.43212890625, "epoch": 0.096, "grad_norm": 0.3507938086986542, "kl": 0.0007605552673339844, "learning_rate": 1.6837835672960831e-07, "loss": 0.0168, "reward": 0.6666666967794299, "reward_std": 0.5004496201872826, "rewards/accuracy_reward": 0.2708333432674408, "rewards/format_reward": 0.3958333386108279, "step": 84, "w_high_ratio": 0.0, "w_low_ratio": 0.05950552970170975, "w_max": 1.3848203122615814, "w_mean": 1.1114588379859924, "w_min": 0.0, "w_std": 0.2529653161764145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3263.2188110351562, "cov_mean": -2.9235156034701504e-05, "cov_std": 0.2883659228682518, "entropy": 0.3173828125, "epoch": 0.09714285714285714, "grad_norm": 0.31331735849380493, "kl": 0.0004665851593017578, "learning_rate": 1.6028856829700258e-07, "loss": 0.0412, "reward": 0.6145833656191826, "reward_std": 0.5151766017079353, "rewards/accuracy_reward": 0.18750000651925802, "rewards/format_reward": 0.4270833469927311, "step": 85, "w_high_ratio": 0.041407618671655655, "w_low_ratio": 0.04611168662086129, "w_max": 1.504797637462616, "w_mean": 1.16913503408432, "w_min": 0.0, "w_std": 0.2160215601325035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3107.416748046875, "cov_mean": 2.2301139551927918e-05, "cov_std": 0.29477884620428085, "entropy": 0.4326171875, "epoch": 0.09828571428571428, "grad_norm": 0.44227737188339233, "kl": 0.0021905899047851562, "learning_rate": 1.5267358321348285e-07, "loss": -0.017, "reward": 0.5312500074505806, "reward_std": 0.2660149596631527, "rewards/accuracy_reward": 0.12500000093132257, "rewards/format_reward": 0.40625, "step": 86, "w_high_ratio": 0.1639367900788784, "w_low_ratio": 0.026534121483564377, "w_max": 2.0416696071624756, "w_mean": 1.30596524477005, "w_min": 0.25, "w_std": 0.21408291161060333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3028.3438110351562, "cov_mean": -7.177443694672547e-05, "cov_std": 0.4208982028067112, "entropy": 0.49169921875, "epoch": 0.09942857142857142, "grad_norm": 0.28011003136634827, "kl": 0.0020780563354492188, "learning_rate": 1.4554267916537495e-07, "loss": -0.0235, "reward": 0.7083333432674408, "reward_std": 0.48819613456726074, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.5000000149011612, "step": 87, "w_high_ratio": 0.04094649478793144, "w_low_ratio": 0.05161830177530646, "w_max": 1.7180204689502716, "w_mean": 1.2395348846912384, "w_min": 1.7816459000041805e-41, "w_std": 0.2497432641685009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2961.7293090820312, "cov_mean": 0.00011648364034044789, "cov_std": 0.5657102465629578, "entropy": 0.44140625, "epoch": 0.10057142857142858, "grad_norm": 0.5410559177398682, "kl": 0.0027256011962890625, "learning_rate": 1.3890454406082956e-07, "loss": -0.0379, "reward": 0.8437500298023224, "reward_std": 0.6055800318717957, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.5520833507180214, "step": 88, "w_high_ratio": 0.15312667191028595, "w_low_ratio": 0.052226919680833817, "w_max": 2.147995412349701, "w_mean": 1.4601677060127258, "w_min": 1.5606149987442183e-34, "w_std": 0.37410108000040054 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3738.166748046875, "cov_mean": -3.068390833504964e-05, "cov_std": 0.2844373546540737, "entropy": 0.41650390625, "epoch": 0.10171428571428572, "grad_norm": 0.2505849003791809, "kl": 0.001811981201171875, "learning_rate": 1.3276726544494571e-07, "loss": -0.0108, "reward": 0.354166679084301, "reward_std": 0.48622245341539383, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.229166679084301, "step": 89, "w_high_ratio": 0.0, "w_low_ratio": 0.038497451692819595, "w_max": 1.4062875509262085, "w_mean": 1.0552209317684174, "w_min": 3.5878705750726184e-40, "w_std": 0.16452663764357567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2751.8541870117188, "cov_mean": 7.335069381042558e-05, "cov_std": 0.3152489699423313, "entropy": 0.5263671875, "epoch": 0.10285714285714286, "grad_norm": 0.4594099223613739, "kl": 0.0036954879760742188, "learning_rate": 1.2713832064634125e-07, "loss": 0.0116, "reward": 0.593750013038516, "reward_std": 0.30915412306785583, "rewards/accuracy_reward": 0.06250000279396772, "rewards/format_reward": 0.5312500176951289, "step": 90, "w_high_ratio": 0.22096561267971992, "w_low_ratio": 0.028408138314262033, "w_max": 2.2681187987327576, "w_mean": 1.453046053647995, "w_min": 0.0, "w_std": 0.17159932479262352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3386.9584350585938, "cov_mean": 5.254377992969239e-05, "cov_std": 0.34420739486813545, "entropy": 0.43603515625, "epoch": 0.104, "grad_norm": 0.40760353207588196, "kl": 0.0015692710876464844, "learning_rate": 1.220245676671809e-07, "loss": -0.0737, "reward": 0.5000000204890966, "reward_std": 0.41079987585544586, "rewards/accuracy_reward": 0.15625000279396772, "rewards/format_reward": 0.3437500102445483, "step": 91, "w_high_ratio": 0.044647760689258575, "w_low_ratio": 0.03610147628933191, "w_max": 1.6350789666175842, "w_mean": 1.218572586774826, "w_min": 0.25, "w_std": 0.22148512117564678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2950.3333740234375, "cov_mean": 4.718162654171465e-05, "cov_std": 0.552252508699894, "entropy": 0.369140625, "epoch": 0.10514285714285715, "grad_norm": 0.6825612783432007, "kl": 0.0022275447845458984, "learning_rate": 1.1743223682775649e-07, "loss": 0.0233, "reward": 0.6354167014360428, "reward_std": 0.5102165751159191, "rewards/accuracy_reward": 0.14583333861082792, "rewards/format_reward": 0.4895833432674408, "step": 92, "w_high_ratio": 0.26809635758399963, "w_low_ratio": 0.05027919542044401, "w_max": 2.5253304839134216, "w_mean": 1.5156863629817963, "w_min": 0.0, "w_std": 0.3864224813878536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3803.7500610351562, "cov_mean": 3.139047112199478e-05, "cov_std": 0.20749856531620026, "entropy": 0.52001953125, "epoch": 0.10628571428571429, "grad_norm": 0.24012291431427002, "kl": 0.0020914077758789062, "learning_rate": 1.1336692317580158e-07, "loss": 0.0268, "reward": 0.0937500037252903, "reward_std": 0.19024791568517685, "rewards/accuracy_reward": 0.010416666977107525, "rewards/format_reward": 0.0833333358168602, "step": 93, "w_high_ratio": 0.0, "w_low_ratio": 0.02966291829943657, "w_max": 1.2947639226913452, "w_mean": 1.0370966792106628, "w_min": 0.5, "w_std": 0.11119646579027176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3297.8646240234375, "cov_mean": 3.7244943996483926e-05, "cov_std": 0.31120600923895836, "entropy": 0.505859375, "epoch": 0.10742857142857143, "grad_norm": 0.19109465181827545, "kl": 0.002716541290283203, "learning_rate": 1.0983357966978745e-07, "loss": 0.0047, "reward": 0.5312500251457095, "reward_std": 0.33500421047210693, "rewards/accuracy_reward": 0.18750000186264515, "rewards/format_reward": 0.3437500102445483, "step": 94, "w_high_ratio": 0.0031294284854084253, "w_low_ratio": 0.0420422088354826, "w_max": 1.7046481668949127, "w_mean": 1.1779425740242004, "w_min": 0.0, "w_std": 0.18928811699151993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3739.1875610351562, "cov_mean": -3.053597947655362e-05, "cov_std": 0.34536129236221313, "entropy": 0.39794921875, "epoch": 0.10857142857142857, "grad_norm": 0.1897294968366623, "kl": 0.00032019615173339844, "learning_rate": 1.068365111445064e-07, "loss": 0.0256, "reward": 0.3125, "reward_std": 0.3973517417907715, "rewards/accuracy_reward": 0.09375, "rewards/format_reward": 0.21875, "step": 95, "w_high_ratio": 0.0, "w_low_ratio": 0.03817740920931101, "w_max": 1.3027753233909607, "w_mean": 1.0644334852695465, "w_min": 0.25, "w_std": 0.16921117901802063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3006.354248046875, "cov_mean": 1.1065521448472282e-05, "cov_std": 0.36543623358011246, "entropy": 0.36279296875, "epoch": 0.10971428571428571, "grad_norm": 0.516417384147644, "kl": 0.002213001251220703, "learning_rate": 1.0437936906629334e-07, "loss": 0.0172, "reward": 0.7395833507180214, "reward_std": 0.40412483364343643, "rewards/accuracy_reward": 0.2604166716337204, "rewards/format_reward": 0.479166679084301, "step": 96, "w_high_ratio": 0.04663441330194473, "w_low_ratio": 0.037586700869724154, "w_max": 1.9617899954319, "w_mean": 1.2268487811088562, "w_min": 3.964865404176085e-40, "w_std": 0.2116919718682766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3466.0209350585938, "cov_mean": -1.2588386653078487e-05, "cov_std": 0.4068885110318661, "entropy": 0.4091796875, "epoch": 0.11085714285714286, "grad_norm": 0.28435367345809937, "kl": 0.0007953643798828125, "learning_rate": 1.0246514708427701e-07, "loss": -0.0236, "reward": 0.5208333563059568, "reward_std": 0.5388510599732399, "rewards/accuracy_reward": 0.19791667722165585, "rewards/format_reward": 0.3229166818782687, "step": 97, "w_high_ratio": 0.0, "w_low_ratio": 0.04683105740696192, "w_max": 1.4916425049304962, "w_mean": 1.1599352061748505, "w_min": 0.0, "w_std": 0.24916893057525158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3058.0209350585938, "cov_mean": 0.00017139434021373745, "cov_std": 0.42342133820056915, "entropy": 0.37744140625, "epoch": 0.112, "grad_norm": 0.39048805832862854, "kl": 0.0005214214324951172, "learning_rate": 1.0109617738307911e-07, "loss": -0.044, "reward": 0.645833358168602, "reward_std": 0.4053402505815029, "rewards/accuracy_reward": 0.1770833432674408, "rewards/format_reward": 0.4687500149011612, "step": 98, "w_high_ratio": 0.04422338306903839, "w_low_ratio": 0.044881118927150965, "w_max": 1.805379331111908, "w_mean": 1.279258906841278, "w_min": 9.581378249820937e-43, "w_std": 0.26856033504009247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 3090.6771545410156, "cov_mean": -1.8359070736551075e-05, "cov_std": 0.19972562417387962, "entropy": 0.34521484375, "epoch": 0.11314285714285714, "grad_norm": 0.27053090929985046, "kl": 0.0011525154113769531, "learning_rate": 1.002741278414069e-07, "loss": 0.0295, "reward": 0.5416666893288493, "reward_std": 0.29046234488487244, "rewards/accuracy_reward": 0.2187500074505806, "rewards/format_reward": 0.3229166669771075, "step": 99, "w_high_ratio": 0.0, "w_low_ratio": 0.028498666360974312, "w_max": 1.280459076166153, "w_mean": 1.0813924968242645, "w_min": 0.25, "w_std": 0.1357164867222309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 2991.7500610351562, "cov_mean": -9.51583083406149e-05, "cov_std": 0.3894283100962639, "entropy": 0.356201171875, "epoch": 0.11428571428571428, "grad_norm": 0.5418879389762878, "kl": 0.00218963623046875, "learning_rate": 1e-07, "loss": -0.0626, "reward": 0.7395833432674408, "reward_std": 0.479395255446434, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.4479166865348816, "step": 100, "w_high_ratio": 0.14099212735891342, "w_low_ratio": 0.036807731725275517, "w_max": 2.043416380882263, "w_mean": 1.3361191749572754, "w_min": 6.562710904496729e-32, "w_std": 0.25866811722517014 }, { "epoch": 0.11428571428571428, "step": 100, "total_flos": 0.0, "train_loss": -0.004950485862791538, "train_runtime": 8385.5538, "train_samples_per_second": 1.145, "train_steps_per_second": 0.012 } ], "logging_steps": 1, "max_steps": 100, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 6, "trial_name": null, "trial_params": null }