| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.11428571428571428, |
| "eval_steps": 500, |
| "global_step": 100, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2700.4271850585938, |
| "cov_mean": -6.0587970438064076e-05, |
| "cov_std": 0.35307812318205833, |
| "entropy": 0.36767578125, |
| "epoch": 0.001142857142857143, |
| "grad_norm": 0.4553219676017761, |
| "kl": 0.0, |
| "learning_rate": 1e-07, |
| "loss": -0.0382, |
| "reward": 0.7604166893288493, |
| "reward_std": 0.4268697127699852, |
| "rewards/accuracy_reward": 0.25000001303851604, |
| "rewards/format_reward": 0.5104166669771075, |
| "step": 1, |
| "w_high_ratio": 0.2200421690940857, |
| "w_low_ratio": 0.03663695091381669, |
| "w_max": 2.1593789756298065, |
| "w_mean": 1.4711343348026276, |
| "w_min": 6.525355682266089e-35, |
| "w_std": 0.2659660503268242 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3127.3958435058594, |
| "cov_mean": -2.155053698515985e-05, |
| "cov_std": 0.310540571808815, |
| "entropy": 0.353515625, |
| "epoch": 0.002285714285714286, |
| "grad_norm": 0.5147930979728699, |
| "kl": 0.0, |
| "learning_rate": 2e-07, |
| "loss": 0.0049, |
| "reward": 0.6458333637565374, |
| "reward_std": 0.4249730706214905, |
| "rewards/accuracy_reward": 0.2812500102445483, |
| "rewards/format_reward": 0.3645833386108279, |
| "step": 2, |
| "w_high_ratio": 0.05183619633316994, |
| "w_low_ratio": 0.036958135198801756, |
| "w_max": 1.8325217366218567, |
| "w_mean": 1.2113382518291473, |
| "w_min": 0.0, |
| "w_std": 0.20957503467798233 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3685.479248046875, |
| "cov_mean": -4.653575570046087e-05, |
| "cov_std": 0.29687076807022095, |
| "entropy": 0.45556640625, |
| "epoch": 0.0034285714285714284, |
| "grad_norm": 0.23412799835205078, |
| "kl": 4.607439041137695e-05, |
| "learning_rate": 3e-07, |
| "loss": 0.0316, |
| "reward": 0.2291666753590107, |
| "reward_std": 0.35217108577489853, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/format_reward": 0.1770833395421505, |
| "step": 3, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03702388517558575, |
| "w_max": 1.4749425947666168, |
| "w_mean": 1.0885904878377914, |
| "w_min": 3.5302726539688593e-39, |
| "w_std": 0.1792612113058567 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2459.854248046875, |
| "cov_mean": 2.6621969936968526e-05, |
| "cov_std": 0.42025458812713623, |
| "entropy": 0.4140625, |
| "epoch": 0.004571428571428572, |
| "grad_norm": 0.42652714252471924, |
| "kl": 3.8504600524902344e-05, |
| "learning_rate": 4e-07, |
| "loss": 0.0096, |
| "reward": 0.7604167014360428, |
| "reward_std": 0.4487093985080719, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/format_reward": 0.6354166865348816, |
| "step": 4, |
| "w_high_ratio": 0.2003328688442707, |
| "w_low_ratio": 0.04321334883570671, |
| "w_max": 2.0572283267974854, |
| "w_mean": 1.4974508583545685, |
| "w_min": 0.0, |
| "w_std": 0.26409733667969704 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3507.5833740234375, |
| "cov_mean": 7.01061562722316e-05, |
| "cov_std": 0.5858492627739906, |
| "entropy": 0.45458984375, |
| "epoch": 0.005714285714285714, |
| "grad_norm": 0.5644172430038452, |
| "kl": 4.00543212890625e-05, |
| "learning_rate": 5e-07, |
| "loss": 0.0229, |
| "reward": 0.4583333507180214, |
| "reward_std": 0.6347778141498566, |
| "rewards/accuracy_reward": 0.09375000093132257, |
| "rewards/format_reward": 0.3645833507180214, |
| "step": 5, |
| "w_high_ratio": 0.04538528248667717, |
| "w_low_ratio": 0.06710020452737808, |
| "w_max": 1.6610480844974518, |
| "w_mean": 1.1760589182376862, |
| "w_min": 5.419854268836891e-39, |
| "w_std": 0.322455570101738 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3396.0104370117188, |
| "cov_mean": 1.043153679347597e-05, |
| "cov_std": 0.3946683406829834, |
| "entropy": 0.4580078125, |
| "epoch": 0.006857142857142857, |
| "grad_norm": 0.47143253684043884, |
| "kl": 4.100799560546875e-05, |
| "learning_rate": 6e-07, |
| "loss": 0.0699, |
| "reward": 0.3125000176951289, |
| "reward_std": 0.33619000762701035, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/format_reward": 0.2812500102445483, |
| "step": 6, |
| "w_high_ratio": 0.044003941118717194, |
| "w_low_ratio": 0.04949501994997263, |
| "w_max": 1.6353994011878967, |
| "w_mean": 1.1387673616409302, |
| "w_min": 0.0, |
| "w_std": 0.24893567338585854 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3345.291748046875, |
| "cov_mean": -7.031985660432838e-06, |
| "cov_std": 0.4864897131919861, |
| "entropy": 0.38818359375, |
| "epoch": 0.008, |
| "grad_norm": 0.2716952860355377, |
| "kl": 2.446770668029785e-05, |
| "learning_rate": 7e-07, |
| "loss": -0.0398, |
| "reward": 0.7395833507180214, |
| "reward_std": 0.5676274970173836, |
| "rewards/accuracy_reward": 0.1979166679084301, |
| "rewards/format_reward": 0.5416666828095913, |
| "step": 7, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.051307737827301025, |
| "w_max": 1.4717772901058197, |
| "w_mean": 1.1606104969978333, |
| "w_min": 0.0, |
| "w_std": 0.2527086026966572 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2934.9583740234375, |
| "cov_mean": -6.692781494166411e-05, |
| "cov_std": 0.31714635342359543, |
| "entropy": 0.36328125, |
| "epoch": 0.009142857142857144, |
| "grad_norm": 0.24377422034740448, |
| "kl": 1.4781951904296875e-05, |
| "learning_rate": 8e-07, |
| "loss": -0.0011, |
| "reward": 0.8750000409781933, |
| "reward_std": 0.5431903004646301, |
| "rewards/accuracy_reward": 0.38541667722165585, |
| "rewards/format_reward": 0.48958336375653744, |
| "step": 8, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03932169498875737, |
| "w_max": 1.5945640802383423, |
| "w_mean": 1.1880076229572296, |
| "w_min": 0.0, |
| "w_std": 0.20366584323346615 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3466.6250610351562, |
| "cov_mean": -5.504875753103988e-05, |
| "cov_std": 0.3686807006597519, |
| "entropy": 0.4501953125, |
| "epoch": 0.010285714285714285, |
| "grad_norm": 0.3517554700374603, |
| "kl": 4.1872262954711914e-05, |
| "learning_rate": 9e-07, |
| "loss": -0.0325, |
| "reward": 0.4375000074505806, |
| "reward_std": 0.5021138191223145, |
| "rewards/accuracy_reward": 0.13541666883975267, |
| "rewards/format_reward": 0.3020833432674408, |
| "step": 9, |
| "w_high_ratio": 0.021586479619145393, |
| "w_low_ratio": 0.04805116541683674, |
| "w_max": 1.7995203733444214, |
| "w_mean": 1.1711468696594238, |
| "w_min": 1.0441714550433176e-38, |
| "w_std": 0.24575592577457428 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2808.8438720703125, |
| "cov_mean": -2.2776248442824e-05, |
| "cov_std": 0.3646387383341789, |
| "entropy": 0.326171875, |
| "epoch": 0.011428571428571429, |
| "grad_norm": 0.44795799255371094, |
| "kl": 2.4825334548950195e-05, |
| "learning_rate": 1e-06, |
| "loss": -0.0625, |
| "reward": 0.6875000223517418, |
| "reward_std": 0.48652123659849167, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/format_reward": 0.4791666828095913, |
| "step": 10, |
| "w_high_ratio": 0.1090739294886589, |
| "w_low_ratio": 0.04266490042209625, |
| "w_max": 2.1747676730155945, |
| "w_mean": 1.3257513344287872, |
| "w_min": 0.0, |
| "w_std": 0.2564368508756161 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3580.1355590820312, |
| "cov_mean": -4.214262844470795e-06, |
| "cov_std": 0.35131024569272995, |
| "entropy": 0.3720703125, |
| "epoch": 0.012571428571428572, |
| "grad_norm": 0.480006605386734, |
| "kl": 3.1560659408569336e-05, |
| "learning_rate": 9.997258721585931e-07, |
| "loss": 0.0246, |
| "reward": 0.3333333507180214, |
| "reward_std": 0.568866454064846, |
| "rewards/accuracy_reward": 0.12500000279396772, |
| "rewards/format_reward": 0.20833334140479565, |
| "step": 11, |
| "w_high_ratio": 0.04230208322405815, |
| "w_low_ratio": 0.05235781241208315, |
| "w_max": 1.6345463395118713, |
| "w_mean": 1.1268693506717682, |
| "w_min": 0.0, |
| "w_std": 0.24075055867433548 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2692.9688110351562, |
| "cov_mean": 3.166104261254077e-05, |
| "cov_std": 0.33987831324338913, |
| "entropy": 0.40576171875, |
| "epoch": 0.013714285714285714, |
| "grad_norm": 0.5367066860198975, |
| "kl": 4.026293754577637e-05, |
| "learning_rate": 9.989038226169207e-07, |
| "loss": 0.0037, |
| "reward": 0.7500000149011612, |
| "reward_std": 0.4257746562361717, |
| "rewards/accuracy_reward": 0.16666667442768812, |
| "rewards/format_reward": 0.5833333469927311, |
| "step": 12, |
| "w_high_ratio": 0.20933324098587036, |
| "w_low_ratio": 0.042826405726373196, |
| "w_max": 2.3706648349761963, |
| "w_mean": 1.5066065192222595, |
| "w_min": 0.0, |
| "w_std": 0.3023643344640732 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3225.3125610351562, |
| "cov_mean": -1.1440834896347951e-05, |
| "cov_std": 0.25972653180360794, |
| "entropy": 0.38330078125, |
| "epoch": 0.014857142857142857, |
| "grad_norm": 0.5191377997398376, |
| "kl": 3.4749507904052734e-05, |
| "learning_rate": 9.975348529157229e-07, |
| "loss": -0.0342, |
| "reward": 0.6354166865348816, |
| "reward_std": 0.3692361190915108, |
| "rewards/accuracy_reward": 0.22916667722165585, |
| "rewards/format_reward": 0.4062500074505806, |
| "step": 13, |
| "w_high_ratio": 0.059321983717381954, |
| "w_low_ratio": 0.02801903593353927, |
| "w_max": 1.86709925532341, |
| "w_mean": 1.2194385528564453, |
| "w_min": 0.0, |
| "w_std": 0.18418971449136734 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3069.2083740234375, |
| "cov_mean": -5.5041197356331395e-05, |
| "cov_std": 0.4479314386844635, |
| "entropy": 0.3857421875, |
| "epoch": 0.016, |
| "grad_norm": 0.4110907018184662, |
| "kl": 3.4496188163757324e-05, |
| "learning_rate": 9.956206309337066e-07, |
| "loss": 0.0262, |
| "reward": 0.6354166716337204, |
| "reward_std": 0.5461946055293083, |
| "rewards/accuracy_reward": 0.20833333861082792, |
| "rewards/format_reward": 0.4270833432674408, |
| "step": 14, |
| "w_high_ratio": 0.05299236997961998, |
| "w_low_ratio": 0.05393570475280285, |
| "w_max": 1.8086846470832825, |
| "w_mean": 1.2297336757183075, |
| "w_min": 0.0, |
| "w_std": 0.29025064408779144 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2969.4063110351562, |
| "cov_mean": 4.094462474313332e-05, |
| "cov_std": 0.28705432265996933, |
| "entropy": 0.372802734375, |
| "epoch": 0.017142857142857144, |
| "grad_norm": 0.2711990475654602, |
| "kl": 2.568960189819336e-05, |
| "learning_rate": 9.931634888554935e-07, |
| "loss": 0.0207, |
| "reward": 0.6666666865348816, |
| "reward_std": 0.326509490609169, |
| "rewards/accuracy_reward": 0.2604166669771075, |
| "rewards/format_reward": 0.4062500111758709, |
| "step": 15, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03214749973267317, |
| "w_max": 1.415438860654831, |
| "w_mean": 1.1301319599151611, |
| "w_min": 0.25, |
| "w_std": 0.17486043274402618 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3889.2083740234375, |
| "cov_mean": 1.429535222996492e-05, |
| "cov_std": 0.1915050856769085, |
| "entropy": 0.46337890625, |
| "epoch": 0.018285714285714287, |
| "grad_norm": 0.2660626173019409, |
| "kl": 3.978610038757324e-05, |
| "learning_rate": 9.901664203302124e-07, |
| "loss": 0.0406, |
| "reward": 0.1250000037252903, |
| "reward_std": 0.24391646683216095, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/format_reward": 0.07291666883975267, |
| "step": 16, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.027141269762068987, |
| "w_max": 1.2118620574474335, |
| "w_mean": 1.0125571489334106, |
| "w_min": 0.25, |
| "w_std": 0.10200810059905052 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2681.927215576172, |
| "cov_mean": 5.457106340145401e-05, |
| "cov_std": 0.4843906760215759, |
| "entropy": 0.46728515625, |
| "epoch": 0.019428571428571427, |
| "grad_norm": 0.5056636929512024, |
| "kl": 4.604458808898926e-05, |
| "learning_rate": 9.866330768241983e-07, |
| "loss": 0.0332, |
| "reward": 0.8020833656191826, |
| "reward_std": 0.5575663819909096, |
| "rewards/accuracy_reward": 0.2500000074505806, |
| "rewards/format_reward": 0.5520833432674408, |
| "step": 17, |
| "w_high_ratio": 0.2290888950228691, |
| "w_low_ratio": 0.04584049433469772, |
| "w_max": 2.3289138078689575, |
| "w_mean": 1.451726257801056, |
| "w_min": 3.2229864679470793e-44, |
| "w_std": 0.33130457252264023 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3172.822998046875, |
| "cov_mean": 5.8477902712184004e-05, |
| "cov_std": 0.3056485205888748, |
| "entropy": 0.36083984375, |
| "epoch": 0.02057142857142857, |
| "grad_norm": 0.5626700520515442, |
| "kl": 2.8699636459350586e-05, |
| "learning_rate": 9.825677631722435e-07, |
| "loss": -0.0633, |
| "reward": 0.5833333507180214, |
| "reward_std": 0.4684683009982109, |
| "rewards/accuracy_reward": 0.15625000279396772, |
| "rewards/format_reward": 0.4270833469927311, |
| "step": 18, |
| "w_high_ratio": 0.05792288854718208, |
| "w_low_ratio": 0.03927373047918081, |
| "w_max": 1.9344473481178284, |
| "w_mean": 1.2674984335899353, |
| "w_min": 0.0, |
| "w_std": 0.20779520832002163 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3062.354217529297, |
| "cov_mean": -3.731243577931309e-05, |
| "cov_std": 0.3898714445531368, |
| "entropy": 0.388671875, |
| "epoch": 0.021714285714285714, |
| "grad_norm": 0.4545780420303345, |
| "kl": 3.999471664428711e-05, |
| "learning_rate": 9.779754323328192e-07, |
| "loss": -0.0387, |
| "reward": 0.8229166772216558, |
| "reward_std": 0.5051202178001404, |
| "rewards/accuracy_reward": 0.37500001583248377, |
| "rewards/format_reward": 0.4479166818782687, |
| "step": 19, |
| "w_high_ratio": 0.01639098348096013, |
| "w_low_ratio": 0.042750393971800804, |
| "w_max": 1.7452828288078308, |
| "w_mean": 1.2359632849693298, |
| "w_min": 0.0, |
| "w_std": 0.25578257627785206 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2637.7813415527344, |
| "cov_mean": -5.605046135315206e-05, |
| "cov_std": 0.43815504014492035, |
| "entropy": 0.339599609375, |
| "epoch": 0.022857142857142857, |
| "grad_norm": 0.3516737222671509, |
| "kl": 4.100799560546875e-05, |
| "learning_rate": 9.728616793536587e-07, |
| "loss": -0.0559, |
| "reward": 1.0104167014360428, |
| "reward_std": 0.5058020055294037, |
| "rewards/accuracy_reward": 0.2916666679084301, |
| "rewards/format_reward": 0.7187500298023224, |
| "step": 20, |
| "w_high_ratio": 0.11945514753460884, |
| "w_low_ratio": 0.04084030445665121, |
| "w_max": 2.0172511637210846, |
| "w_mean": 1.3857647478580475, |
| "w_min": 0.0, |
| "w_std": 0.24064720049500465 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2998.5208740234375, |
| "cov_mean": 2.1053672753623687e-05, |
| "cov_std": 0.19154372438788414, |
| "entropy": 0.423828125, |
| "epoch": 0.024, |
| "grad_norm": 0.40126267075538635, |
| "kl": 0.00011563301086425781, |
| "learning_rate": 9.672327345550543e-07, |
| "loss": 0.0037, |
| "reward": 0.5833333432674408, |
| "reward_std": 0.21227191016077995, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/format_reward": 0.4166666679084301, |
| "step": 21, |
| "w_high_ratio": 0.17914548516273499, |
| "w_low_ratio": 0.0219349917024374, |
| "w_max": 1.8282522857189178, |
| "w_mean": 1.416891485452652, |
| "w_min": 0.25, |
| "w_std": 0.1492762751877308 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1878.3125457763672, |
| "cov_mean": 9.24348714761436e-05, |
| "cov_std": 0.4086475111544132, |
| "entropy": 0.3994140625, |
| "epoch": 0.025142857142857144, |
| "grad_norm": 0.7276642918586731, |
| "kl": 0.00010776519775390625, |
| "learning_rate": 9.610954559391704e-07, |
| "loss": -0.054, |
| "reward": 1.1041667014360428, |
| "reward_std": 0.40466368943452835, |
| "rewards/accuracy_reward": 0.29166667722165585, |
| "rewards/format_reward": 0.8125000298023224, |
| "step": 22, |
| "w_high_ratio": 0.22925379872322083, |
| "w_low_ratio": 0.04057216551154852, |
| "w_max": 2.499152511358261, |
| "w_mean": 1.6791247129440308, |
| "w_min": 0.0, |
| "w_std": 0.29966018348932266 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2871.3438720703125, |
| "cov_mean": -0.0001690889857854927, |
| "cov_std": 0.5799647718667984, |
| "entropy": 0.37353515625, |
| "epoch": 0.026285714285714287, |
| "grad_norm": 0.5503926277160645, |
| "kl": 6.878376007080078e-05, |
| "learning_rate": 9.54457320834625e-07, |
| "loss": -0.0364, |
| "reward": 0.6354166865348816, |
| "reward_std": 0.6411799788475037, |
| "rewards/accuracy_reward": 0.16666667070239782, |
| "rewards/format_reward": 0.4687500149011612, |
| "step": 23, |
| "w_high_ratio": 0.326111301779747, |
| "w_low_ratio": 0.05267245415598154, |
| "w_max": 2.298383414745331, |
| "w_mean": 1.572682112455368, |
| "w_min": 0.0, |
| "w_std": 0.3976207822561264 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2923.572998046875, |
| "cov_mean": 7.625344460393535e-05, |
| "cov_std": 0.3671033903956413, |
| "entropy": 0.3544921875, |
| "epoch": 0.027428571428571427, |
| "grad_norm": 0.37181225419044495, |
| "kl": 7.140636444091797e-05, |
| "learning_rate": 9.473264167865171e-07, |
| "loss": -0.0703, |
| "reward": 0.8541666865348816, |
| "reward_std": 0.5709189921617508, |
| "rewards/accuracy_reward": 0.3020833432674408, |
| "rewards/format_reward": 0.5520833432674408, |
| "step": 24, |
| "w_high_ratio": 0.09629620611667633, |
| "w_low_ratio": 0.037845601327717304, |
| "w_max": 2.1263908445835114, |
| "w_mean": 1.3701403737068176, |
| "w_min": 0.25, |
| "w_std": 0.23599852621555328 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2825.1458740234375, |
| "cov_mean": -5.495093546414864e-05, |
| "cov_std": 0.29340429976582527, |
| "entropy": 0.4248046875, |
| "epoch": 0.02857142857142857, |
| "grad_norm": 0.32156902551651, |
| "kl": 0.00015220046043395996, |
| "learning_rate": 9.397114317029974e-07, |
| "loss": -0.0438, |
| "reward": 0.677083358168602, |
| "reward_std": 0.3831539675593376, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/format_reward": 0.4687500149011612, |
| "step": 25, |
| "w_high_ratio": 0.16885175555944443, |
| "w_low_ratio": 0.032804112415760756, |
| "w_max": 1.8952468037605286, |
| "w_mean": 1.3390689194202423, |
| "w_min": 0.25, |
| "w_std": 0.22369066812098026 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3157.4584350585938, |
| "cov_mean": 3.823744009423535e-06, |
| "cov_std": 0.15900863334536552, |
| "entropy": 0.4033203125, |
| "epoch": 0.029714285714285714, |
| "grad_norm": 0.18318897485733032, |
| "kl": 2.8640031814575195e-05, |
| "learning_rate": 9.316216432703916e-07, |
| "loss": -0.0211, |
| "reward": 0.71875, |
| "reward_std": 0.1498664766550064, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/format_reward": 0.4895833358168602, |
| "step": 26, |
| "w_high_ratio": 0.0625, |
| "w_low_ratio": 0.012461496517062187, |
| "w_max": 1.7156271040439606, |
| "w_mean": 1.2929016053676605, |
| "w_min": 0.5, |
| "w_std": 0.07156859897077084 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3333.5729370117188, |
| "cov_mean": 9.062682693183888e-06, |
| "cov_std": 0.33011873811483383, |
| "entropy": 0.4306640625, |
| "epoch": 0.030857142857142857, |
| "grad_norm": 0.22338484227657318, |
| "kl": 7.772445678710938e-05, |
| "learning_rate": 9.230669076497687e-07, |
| "loss": -0.0295, |
| "reward": 0.5833333507180214, |
| "reward_std": 0.4810289070010185, |
| "rewards/accuracy_reward": 0.15625000558793545, |
| "rewards/format_reward": 0.4270833358168602, |
| "step": 27, |
| "w_high_ratio": 0.125, |
| "w_low_ratio": 0.04519441071897745, |
| "w_max": 1.898216873407364, |
| "w_mean": 1.2821258306503296, |
| "w_min": 0.0, |
| "w_std": 0.1903173178434372 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2990.6563110351562, |
| "cov_mean": -2.9691624149563722e-05, |
| "cov_std": 0.27876436710357666, |
| "entropy": 0.392578125, |
| "epoch": 0.032, |
| "grad_norm": 0.2658036947250366, |
| "kl": 0.00012731552124023438, |
| "learning_rate": 9.140576474687263e-07, |
| "loss": -0.0147, |
| "reward": 0.8229166716337204, |
| "reward_std": 0.32865703850984573, |
| "rewards/accuracy_reward": 0.3437500074505806, |
| "rewards/format_reward": 0.4791666716337204, |
| "step": 28, |
| "w_high_ratio": 0.03581305593252182, |
| "w_low_ratio": 0.036258784122765064, |
| "w_max": 1.8826820850372314, |
| "w_mean": 1.2603062689304352, |
| "w_min": 0.25, |
| "w_std": 0.2064611092209816 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3477.0625610351562, |
| "cov_mean": 3.658079458546126e-05, |
| "cov_std": 0.3980727344751358, |
| "entropy": 0.41162109375, |
| "epoch": 0.03314285714285714, |
| "grad_norm": 0.5845446586608887, |
| "kl": 0.00014966726303100586, |
| "learning_rate": 9.046048391230247e-07, |
| "loss": -0.03, |
| "reward": 0.4479166781529784, |
| "reward_std": 0.4323580041527748, |
| "rewards/accuracy_reward": 0.13541667256504297, |
| "rewards/format_reward": 0.31250000838190317, |
| "step": 29, |
| "w_high_ratio": 0.03906010463833809, |
| "w_low_ratio": 0.04348241724073887, |
| "w_max": 1.5083820223808289, |
| "w_mean": 1.1707176566123962, |
| "w_min": 0.0, |
| "w_std": 0.23769092187285423 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3228.4896240234375, |
| "cov_mean": 8.98454500202206e-05, |
| "cov_std": 0.4796721711754799, |
| "entropy": 0.388671875, |
| "epoch": 0.03428571428571429, |
| "grad_norm": 0.5218952894210815, |
| "kl": 0.0005021095275878906, |
| "learning_rate": 8.9471999940354e-07, |
| "loss": -0.0373, |
| "reward": 0.8229167014360428, |
| "reward_std": 0.6668380200862885, |
| "rewards/accuracy_reward": 0.3020833395421505, |
| "rewards/format_reward": 0.5208333469927311, |
| "step": 30, |
| "w_high_ratio": 0.09154843725264072, |
| "w_low_ratio": 0.050314661115407944, |
| "w_max": 1.7150111198425293, |
| "w_mean": 1.273286134004593, |
| "w_min": 0.0, |
| "w_std": 0.2975125089287758 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3316.479248046875, |
| "cov_mean": -2.0371426671772497e-05, |
| "cov_std": 0.32399024814367294, |
| "entropy": 0.39453125, |
| "epoch": 0.03542857142857143, |
| "grad_norm": 0.26446813344955444, |
| "kl": 0.00034368038177490234, |
| "learning_rate": 8.844151714648274e-07, |
| "loss": 0.0163, |
| "reward": 0.4791666828095913, |
| "reward_std": 0.45351940393447876, |
| "rewards/accuracy_reward": 0.17708333767950535, |
| "rewards/format_reward": 0.30208333395421505, |
| "step": 31, |
| "w_high_ratio": 0.16790161281824112, |
| "w_low_ratio": 0.038946577813476324, |
| "w_max": 1.8747785687446594, |
| "w_mean": 1.287878304719925, |
| "w_min": 0.25, |
| "w_std": 0.18489549681544304 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3449.291748046875, |
| "cov_mean": -4.376452466203773e-05, |
| "cov_std": 0.3894844390451908, |
| "entropy": 0.443359375, |
| "epoch": 0.036571428571428574, |
| "grad_norm": 0.3080126643180847, |
| "kl": 0.0014617443084716797, |
| "learning_rate": 8.737029101523929e-07, |
| "loss": -0.0432, |
| "reward": 0.7083333469927311, |
| "reward_std": 0.5129974186420441, |
| "rewards/accuracy_reward": 0.30208333395421505, |
| "rewards/format_reward": 0.4062500037252903, |
| "step": 32, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04653155663982034, |
| "w_max": 1.6309894919395447, |
| "w_mean": 1.170417159795761, |
| "w_min": 0.0, |
| "w_std": 0.2305326908826828 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3521.125, |
| "cov_mean": -3.824161467491649e-06, |
| "cov_std": 0.3757542073726654, |
| "entropy": 0.380859375, |
| "epoch": 0.037714285714285714, |
| "grad_norm": 0.2623606026172638, |
| "kl": 0.0004220008850097656, |
| "learning_rate": 8.625962667065487e-07, |
| "loss": -0.0075, |
| "reward": 0.5520833507180214, |
| "reward_std": 0.5886104479432106, |
| "rewards/accuracy_reward": 0.1979166679084301, |
| "rewards/format_reward": 0.3541666828095913, |
| "step": 33, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04900518897920847, |
| "w_max": 1.5290310084819794, |
| "w_mean": 1.145493358373642, |
| "w_min": 0.0, |
| "w_std": 0.23011762276291847 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2698.3020935058594, |
| "cov_mean": 5.745048838434741e-06, |
| "cov_std": 0.4045129865407944, |
| "entropy": 0.4296875, |
| "epoch": 0.038857142857142854, |
| "grad_norm": 0.4748375713825226, |
| "kl": 0.0005691051483154297, |
| "learning_rate": 8.511087728614862e-07, |
| "loss": -0.0352, |
| "reward": 0.979166679084301, |
| "reward_std": 0.4912763461470604, |
| "rewards/accuracy_reward": 0.4270833544433117, |
| "rewards/format_reward": 0.5520833386108279, |
| "step": 34, |
| "w_high_ratio": 0.1756632700562477, |
| "w_low_ratio": 0.04408560786396265, |
| "w_max": 2.0611276030540466, |
| "w_mean": 1.413420170545578, |
| "w_min": 6.104692548832184e-37, |
| "w_std": 0.29224658384919167 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3220.1146240234375, |
| "cov_mean": 2.1198811737122014e-06, |
| "cov_std": 0.43479467928409576, |
| "entropy": 0.41845703125, |
| "epoch": 0.04, |
| "grad_norm": 0.34247061610221863, |
| "kl": 0.0007988214492797852, |
| "learning_rate": 8.392544243589427e-07, |
| "loss": 0.0317, |
| "reward": 0.677083358168602, |
| "reward_std": 0.6704623848199844, |
| "rewards/accuracy_reward": 0.2604166753590107, |
| "rewards/format_reward": 0.4166666716337204, |
| "step": 35, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.059821099042892456, |
| "w_max": 1.4533329010009766, |
| "w_mean": 1.1137926578521729, |
| "w_min": 0.0, |
| "w_std": 0.26089195907115936 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3597.8333740234375, |
| "cov_mean": 4.828646342502907e-05, |
| "cov_std": 0.35842880606651306, |
| "entropy": 0.49462890625, |
| "epoch": 0.04114285714285714, |
| "grad_norm": 0.41333624720573425, |
| "kl": 0.00125160813331604, |
| "learning_rate": 8.270476638965461e-07, |
| "loss": 0.0601, |
| "reward": 0.2604166744276881, |
| "reward_std": 0.3833026885986328, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/format_reward": 0.21875000093132257, |
| "step": 36, |
| "w_high_ratio": 0.03994074463844299, |
| "w_low_ratio": 0.04655152838677168, |
| "w_max": 1.508450299501419, |
| "w_mean": 1.11627396941185, |
| "w_min": 0.0, |
| "w_std": 0.19725885801017284 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3524.0625610351562, |
| "cov_mean": -4.1810366383288056e-05, |
| "cov_std": 0.08558216691017151, |
| "entropy": 0.4130859375, |
| "epoch": 0.04228571428571429, |
| "grad_norm": 0.5261622071266174, |
| "kl": 0.0005668401718139648, |
| "learning_rate": 8.145033635316128e-07, |
| "loss": -0.0234, |
| "reward": 0.2291666716337204, |
| "reward_std": 0.07216878235340118, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/format_reward": 0.21875, |
| "step": 37, |
| "w_high_ratio": 0.08015145361423492, |
| "w_low_ratio": 0.006487013306468725, |
| "w_max": 1.6550433039665222, |
| "w_mean": 1.1928074657917023, |
| "w_min": 0.75, |
| "w_std": 0.07643502950668335 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3520.6146240234375, |
| "cov_mean": 2.719235999393277e-05, |
| "cov_std": 0.1502552330493927, |
| "entropy": 0.44580078125, |
| "epoch": 0.04342857142857143, |
| "grad_norm": 0.23012137413024902, |
| "kl": 0.00047528743743896484, |
| "learning_rate": 8.01636806561836e-07, |
| "loss": -0.0058, |
| "reward": 0.364583358168602, |
| "reward_std": 0.22543279081583023, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/format_reward": 0.1979166716337204, |
| "step": 38, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.017723015509545803, |
| "w_max": 1.3341983556747437, |
| "w_mean": 1.0956220924854279, |
| "w_min": 0.5, |
| "w_std": 0.08639993146061897 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3074.104248046875, |
| "cov_mean": 3.08786220557522e-05, |
| "cov_std": 0.2986413836479187, |
| "entropy": 0.35693359375, |
| "epoch": 0.044571428571428574, |
| "grad_norm": 0.19847992062568665, |
| "kl": 0.0011906623840332031, |
| "learning_rate": 7.884636689049422e-07, |
| "loss": -0.0017, |
| "reward": 0.7187500074505806, |
| "reward_std": 0.24960162490606308, |
| "rewards/accuracy_reward": 0.2604166669771075, |
| "rewards/format_reward": 0.4583333432674408, |
| "step": 39, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.030357226729393005, |
| "w_max": 1.4879463911056519, |
| "w_mean": 1.1616290509700775, |
| "w_min": 0.5, |
| "w_std": 0.15971330553293228 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2789.6875610351562, |
| "cov_mean": -5.7511609156790655e-05, |
| "cov_std": 0.31858188658952713, |
| "entropy": 0.38525390625, |
| "epoch": 0.045714285714285714, |
| "grad_norm": 0.2732243835926056, |
| "kl": 0.00257110595703125, |
| "learning_rate": 7.75e-07, |
| "loss": 0.0166, |
| "reward": 0.7500000298023224, |
| "reward_std": 0.41665229201316833, |
| "rewards/accuracy_reward": 0.20833334047347307, |
| "rewards/format_reward": 0.5416666753590107, |
| "step": 40, |
| "w_high_ratio": 0.04059065505862236, |
| "w_low_ratio": 0.037114882841706276, |
| "w_max": 1.8804034292697906, |
| "w_mean": 1.3260886669158936, |
| "w_min": 0.0, |
| "w_std": 0.2053362838923931 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3289.0209350585938, |
| "cov_mean": 1.5465679098269902e-05, |
| "cov_std": 0.45656104385852814, |
| "entropy": 0.37890625, |
| "epoch": 0.046857142857142854, |
| "grad_norm": 0.26020023226737976, |
| "kl": 0.0005598068237304688, |
| "learning_rate": 7.612622032536507e-07, |
| "loss": -0.0115, |
| "reward": 0.489583358168602, |
| "reward_std": 0.507674403488636, |
| "rewards/accuracy_reward": 0.1041666716337204, |
| "rewards/format_reward": 0.3854166865348816, |
| "step": 41, |
| "w_high_ratio": 0.1044064536690712, |
| "w_low_ratio": 0.047306024469435215, |
| "w_max": 2.001980274915695, |
| "w_mean": 1.2991511821746826, |
| "w_min": 3.6180180170197286e-34, |
| "w_std": 0.2728148624300957 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3092.7709350585938, |
| "cov_mean": 4.073379568581004e-05, |
| "cov_std": 0.20339186489582062, |
| "entropy": 0.4619140625, |
| "epoch": 0.048, |
| "grad_norm": 0.27600958943367004, |
| "kl": 0.0006108283996582031, |
| "learning_rate": 7.472670160550848e-07, |
| "loss": 0.0105, |
| "reward": 0.3125000037252903, |
| "reward_std": 0.18993012979626656, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/format_reward": 0.29166666977107525, |
| "step": 42, |
| "w_high_ratio": 0.05672482028603554, |
| "w_low_ratio": 0.01900631980970502, |
| "w_max": 1.7394262850284576, |
| "w_mean": 1.195200800895691, |
| "w_min": 0.25, |
| "w_std": 0.11057326570153236 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3247.92724609375, |
| "cov_mean": 4.6732177906960715e-05, |
| "cov_std": 0.2612141128629446, |
| "entropy": 0.40087890625, |
| "epoch": 0.04914285714285714, |
| "grad_norm": 0.28032609820365906, |
| "kl": 0.001550912857055664, |
| "learning_rate": 7.330314893841101e-07, |
| "loss": -0.0352, |
| "reward": 0.4166666818782687, |
| "reward_std": 0.2987091466784477, |
| "rewards/accuracy_reward": 0.13541666977107525, |
| "rewards/format_reward": 0.2812500027939677, |
| "step": 43, |
| "w_high_ratio": 0.041576627641916275, |
| "w_low_ratio": 0.03293204982765019, |
| "w_max": 1.584185391664505, |
| "w_mean": 1.1372461915016174, |
| "w_min": 0.0, |
| "w_std": 0.16969743371009827 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2958.885482788086, |
| "cov_mean": -4.934846765536349e-05, |
| "cov_std": 0.305513933300972, |
| "entropy": 0.41455078125, |
| "epoch": 0.05028571428571429, |
| "grad_norm": 0.5735454559326172, |
| "kl": 0.001309037208557129, |
| "learning_rate": 7.185729670371604e-07, |
| "loss": -0.0235, |
| "reward": 0.6770833432674408, |
| "reward_std": 0.37915313243865967, |
| "rewards/accuracy_reward": 0.2291666679084301, |
| "rewards/format_reward": 0.4479166716337204, |
| "step": 44, |
| "w_high_ratio": 0.05263275280594826, |
| "w_low_ratio": 0.033476518001407385, |
| "w_max": 1.7282162308692932, |
| "w_mean": 1.2310035228729248, |
| "w_min": 0.25, |
| "w_std": 0.20509596168994904 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3681.7500610351562, |
| "cov_mean": 5.253156632534228e-05, |
| "cov_std": 0.27400747686624527, |
| "entropy": 0.4130859375, |
| "epoch": 0.05142857142857143, |
| "grad_norm": 0.22661462426185608, |
| "kl": 0.001455068588256836, |
| "learning_rate": 7.039090644965509e-07, |
| "loss": 0.0004, |
| "reward": 0.3854166716337204, |
| "reward_std": 0.4831271469593048, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/format_reward": 0.2187500111758709, |
| "step": 45, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.036470845341682434, |
| "w_max": 1.3770877122879028, |
| "w_mean": 1.0782505869865417, |
| "w_min": 0.25, |
| "w_std": 0.16539720818400383 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3533.5209350585938, |
| "cov_mean": 2.201543838964426e-05, |
| "cov_std": 0.20324882864952087, |
| "entropy": 0.4873046875, |
| "epoch": 0.052571428571428575, |
| "grad_norm": 0.42943075299263, |
| "kl": 0.0011663436889648438, |
| "learning_rate": 6.890576474687263e-07, |
| "loss": 0.0747, |
| "reward": 0.18750000558793545, |
| "reward_std": 0.185950867831707, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/format_reward": 0.1666666679084301, |
| "step": 46, |
| "w_high_ratio": 0.10795218497514725, |
| "w_low_ratio": 0.02661217050626874, |
| "w_max": 1.561707764863968, |
| "w_mean": 1.1653469800949097, |
| "w_min": 0.0, |
| "w_std": 0.1572987660765648 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3152.2188720703125, |
| "cov_mean": -8.783125122135971e-05, |
| "cov_std": 0.4741008821874857, |
| "entropy": 0.388671875, |
| "epoch": 0.053714285714285714, |
| "grad_norm": 0.45350003242492676, |
| "kl": 0.000950932502746582, |
| "learning_rate": 6.740368101176495e-07, |
| "loss": -0.0528, |
| "reward": 0.8125000400468707, |
| "reward_std": 0.599571980535984, |
| "rewards/accuracy_reward": 0.3125000111758709, |
| "rewards/format_reward": 0.5000000102445483, |
| "step": 47, |
| "w_high_ratio": 0.09517703950405121, |
| "w_low_ratio": 0.0510673513635993, |
| "w_max": 1.8478549420833588, |
| "w_mean": 1.2849175333976746, |
| "w_min": 0.0, |
| "w_std": 0.2784101627767086 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2912.8334350585938, |
| "cov_mean": 4.9970367399509996e-08, |
| "cov_std": 0.39979860931634903, |
| "entropy": 0.39013671875, |
| "epoch": 0.054857142857142854, |
| "grad_norm": 0.42112770676612854, |
| "kl": 0.00843667984008789, |
| "learning_rate": 6.588648530198504e-07, |
| "loss": -0.012, |
| "reward": 0.7083333469927311, |
| "reward_std": 0.5211797282099724, |
| "rewards/accuracy_reward": 0.25000001303851604, |
| "rewards/format_reward": 0.45833334140479565, |
| "step": 48, |
| "w_high_ratio": 0.1536145769059658, |
| "w_low_ratio": 0.049599490128457546, |
| "w_max": 2.127849966287613, |
| "w_mean": 1.337787538766861, |
| "w_min": 0.0, |
| "w_std": 0.3069620132446289 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2544.666748046875, |
| "cov_mean": -1.3007749203097774e-05, |
| "cov_std": 0.3665553033351898, |
| "entropy": 0.365234375, |
| "epoch": 0.056, |
| "grad_norm": 0.326923668384552, |
| "kl": 0.007767677307128906, |
| "learning_rate": 6.435602608679916e-07, |
| "loss": 0.028, |
| "reward": 0.770833358168602, |
| "reward_std": 0.4556447044014931, |
| "rewards/accuracy_reward": 0.17708333861082792, |
| "rewards/format_reward": 0.5937500149011612, |
| "step": 49, |
| "w_high_ratio": 0.07310386933386326, |
| "w_low_ratio": 0.03729367163032293, |
| "w_max": 1.794796198606491, |
| "w_mean": 1.2659293115139008, |
| "w_min": 6.92190445412429e-34, |
| "w_std": 0.23256080597639084 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3237.2916870117188, |
| "cov_mean": 1.4073829220251355e-05, |
| "cov_std": 0.2659311108291149, |
| "entropy": 0.33642578125, |
| "epoch": 0.05714285714285714, |
| "grad_norm": 0.18902550637722015, |
| "kl": 0.0010194778442382812, |
| "learning_rate": 6.281416799501187e-07, |
| "loss": -0.0195, |
| "reward": 0.645833345130086, |
| "reward_std": 0.36775920540094376, |
| "rewards/accuracy_reward": 0.2916666669771075, |
| "rewards/format_reward": 0.3541666781529784, |
| "step": 50, |
| "w_high_ratio": 0.03271413967013359, |
| "w_low_ratio": 0.030921938829123974, |
| "w_max": 1.3970885872840881, |
| "w_mean": 1.1265143752098083, |
| "w_min": 0.25, |
| "w_std": 0.15340184792876244 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2497.2188720703125, |
| "cov_mean": 0.00010049804541267804, |
| "cov_std": 0.31800365447998047, |
| "entropy": 0.451171875, |
| "epoch": 0.05828571428571429, |
| "grad_norm": 0.3681160509586334, |
| "kl": 0.0043849945068359375, |
| "learning_rate": 6.126278954320294e-07, |
| "loss": 0.0301, |
| "reward": 0.739583358168602, |
| "reward_std": 0.32177041098475456, |
| "rewards/accuracy_reward": 0.2083333358168602, |
| "rewards/format_reward": 0.5312500149011612, |
| "step": 51, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.038371569477021694, |
| "w_max": 1.5697456300258636, |
| "w_mean": 1.1596841514110565, |
| "w_min": 0.25, |
| "w_std": 0.17389511317014694 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3154.15625, |
| "cov_mean": 1.5342720871558413e-08, |
| "cov_std": 0.3799358010292053, |
| "entropy": 0.38623046875, |
| "epoch": 0.05942857142857143, |
| "grad_norm": 0.2101733535528183, |
| "kl": 0.0017533302307128906, |
| "learning_rate": 5.97037808470444e-07, |
| "loss": 0.0285, |
| "reward": 0.6979166697710752, |
| "reward_std": 0.5342019759118557, |
| "rewards/accuracy_reward": 0.2812500074505806, |
| "rewards/format_reward": 0.41666667722165585, |
| "step": 52, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04498880193568766, |
| "w_max": 1.3936763405799866, |
| "w_mean": 1.130004495382309, |
| "w_min": 0.0, |
| "w_std": 0.19680343009531498 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3053.7708740234375, |
| "cov_mean": 1.7406931874575093e-05, |
| "cov_std": 0.46154169738292694, |
| "entropy": 0.421875, |
| "epoch": 0.060571428571428575, |
| "grad_norm": 0.33937692642211914, |
| "kl": 0.001083970069885254, |
| "learning_rate": 5.813904131848564e-07, |
| "loss": -0.0326, |
| "reward": 0.8645833507180214, |
| "reward_std": 0.6376037672162056, |
| "rewards/accuracy_reward": 0.3020833358168602, |
| "rewards/format_reward": 0.5625000074505806, |
| "step": 53, |
| "w_high_ratio": 0.04103509336709976, |
| "w_low_ratio": 0.05415187170729041, |
| "w_max": 1.5957177877426147, |
| "w_mean": 1.1700108051300049, |
| "w_min": 0.0, |
| "w_std": 0.24823076836764812 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2899.4688720703125, |
| "cov_mean": -8.303408776555443e-06, |
| "cov_std": 0.4274276793003082, |
| "entropy": 0.37158203125, |
| "epoch": 0.061714285714285715, |
| "grad_norm": 0.4694461524486542, |
| "kl": 0.0010991096496582031, |
| "learning_rate": 5.657047735161255e-07, |
| "loss": -0.0728, |
| "reward": 1.020833358168602, |
| "reward_std": 0.6003277003765106, |
| "rewards/accuracy_reward": 0.4583333507180214, |
| "rewards/format_reward": 0.5625000223517418, |
| "step": 54, |
| "w_high_ratio": 0.10438984632492065, |
| "w_low_ratio": 0.044082850217819214, |
| "w_max": 1.875702828168869, |
| "w_mean": 1.3796275556087494, |
| "w_min": 0.0, |
| "w_std": 0.28376903757452965 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3251.5626220703125, |
| "cov_mean": 5.693810294360446e-05, |
| "cov_std": 0.32220450788736343, |
| "entropy": 0.39306640625, |
| "epoch": 0.06285714285714286, |
| "grad_norm": 0.3240751326084137, |
| "kl": 0.0013048648834228516, |
| "learning_rate": 5.5e-07, |
| "loss": -0.0123, |
| "reward": 0.5625000149011612, |
| "reward_std": 0.4148038476705551, |
| "rewards/accuracy_reward": 0.20833333674818277, |
| "rewards/format_reward": 0.3541666753590107, |
| "step": 55, |
| "w_high_ratio": 0.06664891541004181, |
| "w_low_ratio": 0.036852192133665085, |
| "w_max": 1.7040621638298035, |
| "w_mean": 1.2043538689613342, |
| "w_min": 0.25, |
| "w_std": 0.2000352405011654 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3137.3750915527344, |
| "cov_mean": 6.871268851682544e-07, |
| "cov_std": 0.27662914246320724, |
| "entropy": 0.38330078125, |
| "epoch": 0.064, |
| "grad_norm": 0.2951548397541046, |
| "kl": 0.0005192756652832031, |
| "learning_rate": 5.342952264838747e-07, |
| "loss": -0.0268, |
| "reward": 0.6770833507180214, |
| "reward_std": 0.280517403036356, |
| "rewards/accuracy_reward": 0.21875, |
| "rewards/format_reward": 0.4583333507180214, |
| "step": 56, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.030007817316800356, |
| "w_max": 1.4776403903961182, |
| "w_mean": 1.1647382080554962, |
| "w_min": 1.7701902850583252e-42, |
| "w_std": 0.1563316509127617 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3544.9896240234375, |
| "cov_mean": 2.435890860397194e-05, |
| "cov_std": 0.3566681556403637, |
| "entropy": 0.31005859375, |
| "epoch": 0.06514285714285714, |
| "grad_norm": 0.5244222283363342, |
| "kl": 0.00036072731018066406, |
| "learning_rate": 5.186095868151436e-07, |
| "loss": -0.0007, |
| "reward": 0.5208333386108279, |
| "reward_std": 0.5608532726764679, |
| "rewards/accuracy_reward": 0.16666667442768812, |
| "rewards/format_reward": 0.3541666818782687, |
| "step": 57, |
| "w_high_ratio": 0.0016966089606285095, |
| "w_low_ratio": 0.046746447682380676, |
| "w_max": 1.4659978747367859, |
| "w_mean": 1.1122615337371826, |
| "w_min": 0.0, |
| "w_std": 0.21731951087713242 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2408.010467529297, |
| "cov_mean": 6.246490556804929e-06, |
| "cov_std": 0.37900903820991516, |
| "entropy": 0.333251953125, |
| "epoch": 0.06628571428571428, |
| "grad_norm": 0.278490275144577, |
| "kl": 0.0038089752197265625, |
| "learning_rate": 5.02962191529556e-07, |
| "loss": 0.0026, |
| "reward": 0.9479167088866234, |
| "reward_std": 0.44713668525218964, |
| "rewards/accuracy_reward": 0.291666672565043, |
| "rewards/format_reward": 0.6562500111758709, |
| "step": 58, |
| "w_high_ratio": 0.10547220706939697, |
| "w_low_ratio": 0.040943967178463936, |
| "w_max": 1.8222440481185913, |
| "w_mean": 1.3768556416034698, |
| "w_min": 3.915851168125891e-30, |
| "w_std": 0.2507038414478302 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3130.7916870117188, |
| "cov_mean": 4.25004100179649e-06, |
| "cov_std": 0.2073173113167286, |
| "entropy": 0.332763671875, |
| "epoch": 0.06742857142857143, |
| "grad_norm": 0.354891836643219, |
| "kl": 0.00041985511779785156, |
| "learning_rate": 4.873721045679706e-07, |
| "loss": 0.0052, |
| "reward": 0.4791666669771075, |
| "reward_std": 0.2661800682544708, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/format_reward": 0.3333333386108279, |
| "step": 59, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.027361327316612005, |
| "w_max": 1.3359772562980652, |
| "w_mean": 1.097507894039154, |
| "w_min": 0.25, |
| "w_std": 0.14073412120342255 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3174.041748046875, |
| "cov_mean": -5.75426925024658e-05, |
| "cov_std": 0.36878733336925507, |
| "entropy": 0.353515625, |
| "epoch": 0.06857142857142857, |
| "grad_norm": 0.23771578073501587, |
| "kl": 0.0018591880798339844, |
| "learning_rate": 4.7185832004988133e-07, |
| "loss": 0.0192, |
| "reward": 0.5416666865348816, |
| "reward_std": 0.4686477780342102, |
| "rewards/accuracy_reward": 0.14583333861082792, |
| "rewards/format_reward": 0.3958333432674408, |
| "step": 60, |
| "w_high_ratio": 0.04001428559422493, |
| "w_low_ratio": 0.03881241288036108, |
| "w_max": 1.5190648436546326, |
| "w_mean": 1.161254346370697, |
| "w_min": 0.0, |
| "w_std": 0.19259942695498466 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3259.229248046875, |
| "cov_mean": 1.2544531273306347e-05, |
| "cov_std": 0.32379359379410744, |
| "entropy": 0.361328125, |
| "epoch": 0.06971428571428571, |
| "grad_norm": 0.3283206522464752, |
| "kl": 0.0008237361907958984, |
| "learning_rate": 4.5643973913200837e-07, |
| "loss": -0.0185, |
| "reward": 0.614583358168602, |
| "reward_std": 0.4510444663465023, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/format_reward": 0.4479166716337204, |
| "step": 61, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04093870846554637, |
| "w_max": 1.5261969566345215, |
| "w_mean": 1.1891083717346191, |
| "w_min": 0.25, |
| "w_std": 0.2139909453690052 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2809.1771850585938, |
| "cov_mean": 3.0659541152999736e-05, |
| "cov_std": 0.5087622255086899, |
| "entropy": 0.326904296875, |
| "epoch": 0.07085714285714285, |
| "grad_norm": 0.6958292722702026, |
| "kl": 0.022649765014648438, |
| "learning_rate": 4.4113514698014953e-07, |
| "loss": -0.0085, |
| "reward": 0.8958333656191826, |
| "reward_std": 0.6562541723251343, |
| "rewards/accuracy_reward": 0.28125000558793545, |
| "rewards/format_reward": 0.6145833507180214, |
| "step": 62, |
| "w_high_ratio": 0.058254461735486984, |
| "w_low_ratio": 0.053510635159909725, |
| "w_max": 1.9464432299137115, |
| "w_mean": 1.336916297674179, |
| "w_min": 0.0, |
| "w_std": 0.31536465138196945 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2338.1250610351562, |
| "cov_mean": -3.693009693961358e-05, |
| "cov_std": 0.37869442999362946, |
| "entropy": 0.3642578125, |
| "epoch": 0.072, |
| "grad_norm": 0.3641963601112366, |
| "kl": 0.0020456314086914062, |
| "learning_rate": 4.2596318988235037e-07, |
| "loss": -0.0179, |
| "reward": 1.1562500596046448, |
| "reward_std": 0.517615370452404, |
| "rewards/accuracy_reward": 0.4062500074505806, |
| "rewards/format_reward": 0.7500000223517418, |
| "step": 63, |
| "w_high_ratio": 0.10371162719093263, |
| "w_low_ratio": 0.03615910932421684, |
| "w_max": 2.5228151381015778, |
| "w_mean": 1.538686066865921, |
| "w_min": 4.8989017643768394e-36, |
| "w_std": 0.2665465883910656 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3296.385528564453, |
| "cov_mean": -5.260490161163034e-05, |
| "cov_std": 0.32132588326931, |
| "entropy": 0.416015625, |
| "epoch": 0.07314285714285715, |
| "grad_norm": 0.26374608278274536, |
| "kl": 0.0010199546813964844, |
| "learning_rate": 4.1094235253127374e-07, |
| "loss": -0.0008, |
| "reward": 0.6145833656191826, |
| "reward_std": 0.4967978894710541, |
| "rewards/accuracy_reward": 0.22916667722165585, |
| "rewards/format_reward": 0.3854166716337204, |
| "step": 64, |
| "w_high_ratio": 0.125, |
| "w_low_ratio": 0.039727743715047836, |
| "w_max": 1.6750397682189941, |
| "w_mean": 1.3110667765140533, |
| "w_min": 7.912817135887769e-41, |
| "w_std": 0.19634342193603516 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3007.2500610351562, |
| "cov_mean": 3.157142418785952e-05, |
| "cov_std": 0.27999811992049217, |
| "entropy": 0.32763671875, |
| "epoch": 0.07428571428571429, |
| "grad_norm": 0.18390174210071564, |
| "kl": 0.0012006759643554688, |
| "learning_rate": 3.9609093550344907e-07, |
| "loss": 0.0048, |
| "reward": 0.6562500074505806, |
| "reward_std": 0.3042946420609951, |
| "rewards/accuracy_reward": 0.1875, |
| "rewards/format_reward": 0.4687500074505806, |
| "step": 65, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03495390526950359, |
| "w_max": 1.4882625937461853, |
| "w_mean": 1.1612237989902496, |
| "w_min": 0.0, |
| "w_std": 0.14175301045179367 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2299.2396240234375, |
| "cov_mean": 2.2165444875099638e-05, |
| "cov_std": 0.1351991407573223, |
| "entropy": 0.306640625, |
| "epoch": 0.07542857142857143, |
| "grad_norm": 0.42320746183395386, |
| "kl": 0.0011644363403320312, |
| "learning_rate": 3.8142703296283953e-07, |
| "loss": -0.0144, |
| "reward": 0.8541666865348816, |
| "reward_std": 0.1657295972108841, |
| "rewards/accuracy_reward": 0.354166679084301, |
| "rewards/format_reward": 0.5, |
| "step": 66, |
| "w_high_ratio": 0.0625, |
| "w_low_ratio": 0.015947649022564292, |
| "w_max": 1.6368336379528046, |
| "w_mean": 1.2440811097621918, |
| "w_min": 0.25, |
| "w_std": 0.09564665146172047 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3708.572998046875, |
| "cov_mean": 9.353170185022464e-06, |
| "cov_std": 0.17532559856772423, |
| "entropy": 0.3408203125, |
| "epoch": 0.07657142857142857, |
| "grad_norm": 0.18465003371238708, |
| "kl": 0.0010061264038085938, |
| "learning_rate": 3.6696851061588994e-07, |
| "loss": 0.0047, |
| "reward": 0.20833333861082792, |
| "reward_std": 0.1927690953016281, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/format_reward": 0.1666666716337204, |
| "step": 67, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.021573648788034916, |
| "w_max": 1.3159178793430328, |
| "w_mean": 1.0860951095819473, |
| "w_min": 0.25, |
| "w_std": 0.11260665208101273 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2515.729232788086, |
| "cov_mean": 1.1593250746955164e-05, |
| "cov_std": 0.31169629842042923, |
| "entropy": 0.4072265625, |
| "epoch": 0.07771428571428571, |
| "grad_norm": 0.39554792642593384, |
| "kl": 0.009487152099609375, |
| "learning_rate": 3.5273298394491515e-07, |
| "loss": -0.0017, |
| "reward": 0.7812500298023224, |
| "reward_std": 0.47302668541669846, |
| "rewards/accuracy_reward": 0.2187500074505806, |
| "rewards/format_reward": 0.5625000074505806, |
| "step": 68, |
| "w_high_ratio": 0.20789940655231476, |
| "w_low_ratio": 0.03476168913766742, |
| "w_max": 2.3950938880443573, |
| "w_mean": 1.504720538854599, |
| "w_min": 0.25, |
| "w_std": 0.20618349313735962 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2761.9896240234375, |
| "cov_mean": -1.7451284293201752e-05, |
| "cov_std": 0.2980487421154976, |
| "entropy": 0.43212890625, |
| "epoch": 0.07885714285714286, |
| "grad_norm": 0.39313724637031555, |
| "kl": 0.003513336181640625, |
| "learning_rate": 3.387377967463493e-07, |
| "loss": -0.0408, |
| "reward": 0.4791666828095913, |
| "reward_std": 0.3629096820950508, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/format_reward": 0.4375000260770321, |
| "step": 69, |
| "w_high_ratio": 0.06426303647458553, |
| "w_low_ratio": 0.03391136787831783, |
| "w_max": 1.8445489406585693, |
| "w_mean": 1.2533070147037506, |
| "w_min": 0.0, |
| "w_std": 0.1965021900832653 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3302.885498046875, |
| "cov_mean": 2.7586031137616374e-05, |
| "cov_std": 0.25518757849931717, |
| "entropy": 0.35498046875, |
| "epoch": 0.08, |
| "grad_norm": 0.21536409854888916, |
| "kl": 0.0017971992492675781, |
| "learning_rate": 3.250000000000001e-07, |
| "loss": 0.0226, |
| "reward": 0.489583358168602, |
| "reward_std": 0.3120904117822647, |
| "rewards/accuracy_reward": 0.09375000279396772, |
| "rewards/format_reward": 0.3958333432674408, |
| "step": 70, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.035551197826862335, |
| "w_max": 1.2505681812763214, |
| "w_mean": 1.0716167390346527, |
| "w_min": 0.25, |
| "w_std": 0.13985217362642288 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2877.8958740234375, |
| "cov_mean": -1.1374457244528458e-05, |
| "cov_std": 0.2578735649585724, |
| "entropy": 0.40185546875, |
| "epoch": 0.08114285714285714, |
| "grad_norm": 0.5054724812507629, |
| "kl": 0.0020956993103027344, |
| "learning_rate": 3.115363310950578e-07, |
| "loss": 0.0119, |
| "reward": 0.6145833637565374, |
| "reward_std": 0.24616123735904694, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/format_reward": 0.38541666977107525, |
| "step": 71, |
| "w_high_ratio": 0.11903085745871067, |
| "w_low_ratio": 0.02461471315473318, |
| "w_max": 1.9073027968406677, |
| "w_mean": 1.2823512256145477, |
| "w_min": 0.25, |
| "w_std": 0.20722979307174683 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3255.104248046875, |
| "cov_mean": 2.442240383970784e-05, |
| "cov_std": 0.49785537272691727, |
| "entropy": 0.4853515625, |
| "epoch": 0.08228571428571428, |
| "grad_norm": 0.4614424407482147, |
| "kl": 0.004761219024658203, |
| "learning_rate": 2.9836319343816397e-07, |
| "loss": 0.002, |
| "reward": 0.4583333358168602, |
| "reward_std": 0.5298330634832382, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/format_reward": 0.3958333432674408, |
| "step": 72, |
| "w_high_ratio": 0.01689874939620495, |
| "w_low_ratio": 0.058014022186398506, |
| "w_max": 1.7353839576244354, |
| "w_mean": 1.2576677203178406, |
| "w_min": 0.0, |
| "w_std": 0.312557902187109 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3811.3541870117188, |
| "cov_mean": 9.231509466189891e-06, |
| "cov_std": 0.21682696044445038, |
| "entropy": 0.439453125, |
| "epoch": 0.08342857142857144, |
| "grad_norm": 0.11269888281822205, |
| "kl": 0.0003600120544433594, |
| "learning_rate": 2.854966364683872e-07, |
| "loss": 0.0008, |
| "reward": 0.3437500149011612, |
| "reward_std": 0.34103409945964813, |
| "rewards/accuracy_reward": 0.1354166716337204, |
| "rewards/format_reward": 0.2083333432674408, |
| "step": 73, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.025642000138759613, |
| "w_max": 1.2365370392799377, |
| "w_mean": 1.0363657176494598, |
| "w_min": 0.25, |
| "w_std": 0.10616981983184814 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3564.8750610351562, |
| "cov_mean": 7.751962675683899e-05, |
| "cov_std": 0.2809390053153038, |
| "entropy": 0.41064453125, |
| "epoch": 0.08457142857142858, |
| "grad_norm": 0.23569634556770325, |
| "kl": 0.0016646385192871094, |
| "learning_rate": 2.729523361034538e-07, |
| "loss": 0.0156, |
| "reward": 0.479166679084301, |
| "reward_std": 0.4482262283563614, |
| "rewards/accuracy_reward": 0.21875000186264515, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 74, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04224945977330208, |
| "w_max": 1.3714891374111176, |
| "w_mean": 1.06686469912529, |
| "w_min": 0.25, |
| "w_std": 0.17370857670903206 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3226.3126220703125, |
| "cov_mean": 6.714556593578891e-05, |
| "cov_std": 0.3720974437892437, |
| "entropy": 0.36962890625, |
| "epoch": 0.08571428571428572, |
| "grad_norm": 0.22486244142055511, |
| "kl": 0.0015187263488769531, |
| "learning_rate": 2.6074557564105724e-07, |
| "loss": 0.0805, |
| "reward": 0.6041666939854622, |
| "reward_std": 0.3942909240722656, |
| "rewards/accuracy_reward": 0.1979166716337204, |
| "rewards/format_reward": 0.40625001303851604, |
| "step": 75, |
| "w_high_ratio": 0.05757378041744232, |
| "w_low_ratio": 0.045274180360138416, |
| "w_max": 1.8611223101615906, |
| "w_mean": 1.1981039345264435, |
| "w_min": 0.0, |
| "w_std": 0.2294696494936943 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3215.9271850585938, |
| "cov_mean": 1.099553310268675e-05, |
| "cov_std": 0.2080927938222885, |
| "entropy": 0.3857421875, |
| "epoch": 0.08685714285714285, |
| "grad_norm": 0.12048080563545227, |
| "kl": 0.0004360675811767578, |
| "learning_rate": 2.488912271385139e-07, |
| "loss": -0.0073, |
| "reward": 0.4791666865348816, |
| "reward_std": 0.2375478893518448, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/format_reward": 0.4166666865348816, |
| "step": 76, |
| "w_high_ratio": 0.1249999962747097, |
| "w_low_ratio": 0.027999846264719963, |
| "w_max": 1.7428039014339447, |
| "w_mean": 1.2136583924293518, |
| "w_min": 0.25, |
| "w_std": 0.10646160691976547 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3392.4583740234375, |
| "cov_mean": -4.415775742927508e-05, |
| "cov_std": 0.3749671131372452, |
| "entropy": 0.39599609375, |
| "epoch": 0.088, |
| "grad_norm": 0.2782253324985504, |
| "kl": 0.0008301734924316406, |
| "learning_rate": 2.374037332934512e-07, |
| "loss": -0.016, |
| "reward": 0.48958333395421505, |
| "reward_std": 0.39483094960451126, |
| "rewards/accuracy_reward": 0.1354166716337204, |
| "rewards/format_reward": 0.35416667722165585, |
| "step": 77, |
| "w_high_ratio": 0.0396752767264843, |
| "w_low_ratio": 0.03982406808063388, |
| "w_max": 1.5143637657165527, |
| "w_mean": 1.1384893357753754, |
| "w_min": 0.0, |
| "w_std": 0.20439331606030464 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3471.21875, |
| "cov_mean": -2.8236715252205613e-05, |
| "cov_std": 0.4440799057483673, |
| "entropy": 0.36767578125, |
| "epoch": 0.08914285714285715, |
| "grad_norm": 0.34334930777549744, |
| "kl": 0.00042557716369628906, |
| "learning_rate": 2.2629708984760706e-07, |
| "loss": 0.0139, |
| "reward": 0.6041666902601719, |
| "reward_std": 0.5486822947859764, |
| "rewards/accuracy_reward": 0.2708333358168602, |
| "rewards/format_reward": 0.33333334513008595, |
| "step": 78, |
| "w_high_ratio": 0.05815339833498001, |
| "w_low_ratio": 0.04555406514555216, |
| "w_max": 1.7224481999874115, |
| "w_mean": 1.199266105890274, |
| "w_min": 0.0, |
| "w_std": 0.2618200555443764 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2512.6146697998047, |
| "cov_mean": 4.2871850382653065e-05, |
| "cov_std": 0.2797487545758486, |
| "entropy": 0.332275390625, |
| "epoch": 0.09028571428571429, |
| "grad_norm": 0.3756020963191986, |
| "kl": 0.0013785362243652344, |
| "learning_rate": 2.1558482853517253e-07, |
| "loss": 0.0565, |
| "reward": 0.7604166865348816, |
| "reward_std": 0.3231862857937813, |
| "rewards/accuracy_reward": 0.20833333861082792, |
| "rewards/format_reward": 0.5520833432674408, |
| "step": 79, |
| "w_high_ratio": 0.05819739028811455, |
| "w_low_ratio": 0.030684792436659336, |
| "w_max": 2.2010356187820435, |
| "w_mean": 1.3371951580047607, |
| "w_min": 0.0, |
| "w_std": 0.20669112168252468 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3588.604248046875, |
| "cov_mean": -7.164392627601046e-06, |
| "cov_std": 0.2609938234090805, |
| "entropy": 0.46240234375, |
| "epoch": 0.09142857142857143, |
| "grad_norm": 0.20949065685272217, |
| "kl": 0.0011081695556640625, |
| "learning_rate": 2.0528000059645995e-07, |
| "loss": 0.0063, |
| "reward": 0.500000013038516, |
| "reward_std": 0.4404493272304535, |
| "rewards/accuracy_reward": 0.17708333488553762, |
| "rewards/format_reward": 0.3229166781529784, |
| "step": 80, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03598626330494881, |
| "w_max": 1.3815899193286896, |
| "w_mean": 1.1029056310653687, |
| "w_min": 0.25, |
| "w_std": 0.14113666117191315 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3374.7188110351562, |
| "cov_mean": 3.345898390705315e-05, |
| "cov_std": 0.17040981724858284, |
| "entropy": 0.51806640625, |
| "epoch": 0.09257142857142857, |
| "grad_norm": 0.16646912693977356, |
| "kl": 0.0033516883850097656, |
| "learning_rate": 1.9539516087697517e-07, |
| "loss": -0.0106, |
| "reward": 0.3750000102445483, |
| "reward_std": 0.21947956830263138, |
| "rewards/accuracy_reward": 0.09375000558793545, |
| "rewards/format_reward": 0.2812500027939677, |
| "step": 81, |
| "w_high_ratio": 0.0625, |
| "w_low_ratio": 0.021185603458434343, |
| "w_max": 1.6720820665359497, |
| "w_mean": 1.217385858297348, |
| "w_min": 0.25, |
| "w_std": 0.0957618746906519 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3105.322998046875, |
| "cov_mean": -3.050990926567465e-05, |
| "cov_std": 0.23843222856521606, |
| "entropy": 0.42138671875, |
| "epoch": 0.09371428571428571, |
| "grad_norm": 0.34804272651672363, |
| "kl": 0.0018186569213867188, |
| "learning_rate": 1.8594235253127372e-07, |
| "loss": -0.0113, |
| "reward": 0.6458333507180214, |
| "reward_std": 0.31302163377404213, |
| "rewards/accuracy_reward": 0.2500000037252903, |
| "rewards/format_reward": 0.3958333395421505, |
| "step": 82, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.02489804197102785, |
| "w_max": 1.6160834431648254, |
| "w_mean": 1.1803554594516754, |
| "w_min": 0.0, |
| "w_std": 0.15118960477411747 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3147.4583740234375, |
| "cov_mean": -2.670003959792666e-05, |
| "cov_std": 0.2024751603603363, |
| "entropy": 0.43408203125, |
| "epoch": 0.09485714285714286, |
| "grad_norm": 0.24946807324886322, |
| "kl": 0.0009236335754394531, |
| "learning_rate": 1.7693309235023127e-07, |
| "loss": 0.0343, |
| "reward": 0.3854166679084301, |
| "reward_std": 0.24196770787239075, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/format_reward": 0.28125000558793545, |
| "step": 83, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.030171910300850868, |
| "w_max": 1.2905269861221313, |
| "w_mean": 1.0636517405509949, |
| "w_min": 0.25, |
| "w_std": 0.1290601659566164 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3317.5625, |
| "cov_mean": 4.104694835405098e-05, |
| "cov_std": 0.4057937487959862, |
| "entropy": 0.43212890625, |
| "epoch": 0.096, |
| "grad_norm": 0.3507938086986542, |
| "kl": 0.0007605552673339844, |
| "learning_rate": 1.6837835672960831e-07, |
| "loss": 0.0168, |
| "reward": 0.6666666967794299, |
| "reward_std": 0.5004496201872826, |
| "rewards/accuracy_reward": 0.2708333432674408, |
| "rewards/format_reward": 0.3958333386108279, |
| "step": 84, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.05950552970170975, |
| "w_max": 1.3848203122615814, |
| "w_mean": 1.1114588379859924, |
| "w_min": 0.0, |
| "w_std": 0.2529653161764145 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3263.2188110351562, |
| "cov_mean": -2.9235156034701504e-05, |
| "cov_std": 0.2883659228682518, |
| "entropy": 0.3173828125, |
| "epoch": 0.09714285714285714, |
| "grad_norm": 0.31331735849380493, |
| "kl": 0.0004665851593017578, |
| "learning_rate": 1.6028856829700258e-07, |
| "loss": 0.0412, |
| "reward": 0.6145833656191826, |
| "reward_std": 0.5151766017079353, |
| "rewards/accuracy_reward": 0.18750000651925802, |
| "rewards/format_reward": 0.4270833469927311, |
| "step": 85, |
| "w_high_ratio": 0.041407618671655655, |
| "w_low_ratio": 0.04611168662086129, |
| "w_max": 1.504797637462616, |
| "w_mean": 1.16913503408432, |
| "w_min": 0.0, |
| "w_std": 0.2160215601325035 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3107.416748046875, |
| "cov_mean": 2.2301139551927918e-05, |
| "cov_std": 0.29477884620428085, |
| "entropy": 0.4326171875, |
| "epoch": 0.09828571428571428, |
| "grad_norm": 0.44227737188339233, |
| "kl": 0.0021905899047851562, |
| "learning_rate": 1.5267358321348285e-07, |
| "loss": -0.017, |
| "reward": 0.5312500074505806, |
| "reward_std": 0.2660149596631527, |
| "rewards/accuracy_reward": 0.12500000093132257, |
| "rewards/format_reward": 0.40625, |
| "step": 86, |
| "w_high_ratio": 0.1639367900788784, |
| "w_low_ratio": 0.026534121483564377, |
| "w_max": 2.0416696071624756, |
| "w_mean": 1.30596524477005, |
| "w_min": 0.25, |
| "w_std": 0.21408291161060333 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3028.3438110351562, |
| "cov_mean": -7.177443694672547e-05, |
| "cov_std": 0.4208982028067112, |
| "entropy": 0.49169921875, |
| "epoch": 0.09942857142857142, |
| "grad_norm": 0.28011003136634827, |
| "kl": 0.0020780563354492188, |
| "learning_rate": 1.4554267916537495e-07, |
| "loss": -0.0235, |
| "reward": 0.7083333432674408, |
| "reward_std": 0.48819613456726074, |
| "rewards/accuracy_reward": 0.2083333358168602, |
| "rewards/format_reward": 0.5000000149011612, |
| "step": 87, |
| "w_high_ratio": 0.04094649478793144, |
| "w_low_ratio": 0.05161830177530646, |
| "w_max": 1.7180204689502716, |
| "w_mean": 1.2395348846912384, |
| "w_min": 1.7816459000041805e-41, |
| "w_std": 0.2497432641685009 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2961.7293090820312, |
| "cov_mean": 0.00011648364034044789, |
| "cov_std": 0.5657102465629578, |
| "entropy": 0.44140625, |
| "epoch": 0.10057142857142858, |
| "grad_norm": 0.5410559177398682, |
| "kl": 0.0027256011962890625, |
| "learning_rate": 1.3890454406082956e-07, |
| "loss": -0.0379, |
| "reward": 0.8437500298023224, |
| "reward_std": 0.6055800318717957, |
| "rewards/accuracy_reward": 0.2916666753590107, |
| "rewards/format_reward": 0.5520833507180214, |
| "step": 88, |
| "w_high_ratio": 0.15312667191028595, |
| "w_low_ratio": 0.052226919680833817, |
| "w_max": 2.147995412349701, |
| "w_mean": 1.4601677060127258, |
| "w_min": 1.5606149987442183e-34, |
| "w_std": 0.37410108000040054 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3738.166748046875, |
| "cov_mean": -3.068390833504964e-05, |
| "cov_std": 0.2844373546540737, |
| "entropy": 0.41650390625, |
| "epoch": 0.10171428571428572, |
| "grad_norm": 0.2505849003791809, |
| "kl": 0.001811981201171875, |
| "learning_rate": 1.3276726544494571e-07, |
| "loss": -0.0108, |
| "reward": 0.354166679084301, |
| "reward_std": 0.48622245341539383, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/format_reward": 0.229166679084301, |
| "step": 89, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.038497451692819595, |
| "w_max": 1.4062875509262085, |
| "w_mean": 1.0552209317684174, |
| "w_min": 3.5878705750726184e-40, |
| "w_std": 0.16452663764357567 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2751.8541870117188, |
| "cov_mean": 7.335069381042558e-05, |
| "cov_std": 0.3152489699423313, |
| "entropy": 0.5263671875, |
| "epoch": 0.10285714285714286, |
| "grad_norm": 0.4594099223613739, |
| "kl": 0.0036954879760742188, |
| "learning_rate": 1.2713832064634125e-07, |
| "loss": 0.0116, |
| "reward": 0.593750013038516, |
| "reward_std": 0.30915412306785583, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/format_reward": 0.5312500176951289, |
| "step": 90, |
| "w_high_ratio": 0.22096561267971992, |
| "w_low_ratio": 0.028408138314262033, |
| "w_max": 2.2681187987327576, |
| "w_mean": 1.453046053647995, |
| "w_min": 0.0, |
| "w_std": 0.17159932479262352 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3386.9584350585938, |
| "cov_mean": 5.254377992969239e-05, |
| "cov_std": 0.34420739486813545, |
| "entropy": 0.43603515625, |
| "epoch": 0.104, |
| "grad_norm": 0.40760353207588196, |
| "kl": 0.0015692710876464844, |
| "learning_rate": 1.220245676671809e-07, |
| "loss": -0.0737, |
| "reward": 0.5000000204890966, |
| "reward_std": 0.41079987585544586, |
| "rewards/accuracy_reward": 0.15625000279396772, |
| "rewards/format_reward": 0.3437500102445483, |
| "step": 91, |
| "w_high_ratio": 0.044647760689258575, |
| "w_low_ratio": 0.03610147628933191, |
| "w_max": 1.6350789666175842, |
| "w_mean": 1.218572586774826, |
| "w_min": 0.25, |
| "w_std": 0.22148512117564678 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2950.3333740234375, |
| "cov_mean": 4.718162654171465e-05, |
| "cov_std": 0.552252508699894, |
| "entropy": 0.369140625, |
| "epoch": 0.10514285714285715, |
| "grad_norm": 0.6825612783432007, |
| "kl": 0.0022275447845458984, |
| "learning_rate": 1.1743223682775649e-07, |
| "loss": 0.0233, |
| "reward": 0.6354167014360428, |
| "reward_std": 0.5102165751159191, |
| "rewards/accuracy_reward": 0.14583333861082792, |
| "rewards/format_reward": 0.4895833432674408, |
| "step": 92, |
| "w_high_ratio": 0.26809635758399963, |
| "w_low_ratio": 0.05027919542044401, |
| "w_max": 2.5253304839134216, |
| "w_mean": 1.5156863629817963, |
| "w_min": 0.0, |
| "w_std": 0.3864224813878536 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3803.7500610351562, |
| "cov_mean": 3.139047112199478e-05, |
| "cov_std": 0.20749856531620026, |
| "entropy": 0.52001953125, |
| "epoch": 0.10628571428571429, |
| "grad_norm": 0.24012291431427002, |
| "kl": 0.0020914077758789062, |
| "learning_rate": 1.1336692317580158e-07, |
| "loss": 0.0268, |
| "reward": 0.0937500037252903, |
| "reward_std": 0.19024791568517685, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/format_reward": 0.0833333358168602, |
| "step": 93, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.02966291829943657, |
| "w_max": 1.2947639226913452, |
| "w_mean": 1.0370966792106628, |
| "w_min": 0.5, |
| "w_std": 0.11119646579027176 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3297.8646240234375, |
| "cov_mean": 3.7244943996483926e-05, |
| "cov_std": 0.31120600923895836, |
| "entropy": 0.505859375, |
| "epoch": 0.10742857142857143, |
| "grad_norm": 0.19109465181827545, |
| "kl": 0.002716541290283203, |
| "learning_rate": 1.0983357966978745e-07, |
| "loss": 0.0047, |
| "reward": 0.5312500251457095, |
| "reward_std": 0.33500421047210693, |
| "rewards/accuracy_reward": 0.18750000186264515, |
| "rewards/format_reward": 0.3437500102445483, |
| "step": 94, |
| "w_high_ratio": 0.0031294284854084253, |
| "w_low_ratio": 0.0420422088354826, |
| "w_max": 1.7046481668949127, |
| "w_mean": 1.1779425740242004, |
| "w_min": 0.0, |
| "w_std": 0.18928811699151993 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3739.1875610351562, |
| "cov_mean": -3.053597947655362e-05, |
| "cov_std": 0.34536129236221313, |
| "entropy": 0.39794921875, |
| "epoch": 0.10857142857142857, |
| "grad_norm": 0.1897294968366623, |
| "kl": 0.00032019615173339844, |
| "learning_rate": 1.068365111445064e-07, |
| "loss": 0.0256, |
| "reward": 0.3125, |
| "reward_std": 0.3973517417907715, |
| "rewards/accuracy_reward": 0.09375, |
| "rewards/format_reward": 0.21875, |
| "step": 95, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03817740920931101, |
| "w_max": 1.3027753233909607, |
| "w_mean": 1.0644334852695465, |
| "w_min": 0.25, |
| "w_std": 0.16921117901802063 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3006.354248046875, |
| "cov_mean": 1.1065521448472282e-05, |
| "cov_std": 0.36543623358011246, |
| "entropy": 0.36279296875, |
| "epoch": 0.10971428571428571, |
| "grad_norm": 0.516417384147644, |
| "kl": 0.002213001251220703, |
| "learning_rate": 1.0437936906629334e-07, |
| "loss": 0.0172, |
| "reward": 0.7395833507180214, |
| "reward_std": 0.40412483364343643, |
| "rewards/accuracy_reward": 0.2604166716337204, |
| "rewards/format_reward": 0.479166679084301, |
| "step": 96, |
| "w_high_ratio": 0.04663441330194473, |
| "w_low_ratio": 0.037586700869724154, |
| "w_max": 1.9617899954319, |
| "w_mean": 1.2268487811088562, |
| "w_min": 3.964865404176085e-40, |
| "w_std": 0.2116919718682766 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3466.0209350585938, |
| "cov_mean": -1.2588386653078487e-05, |
| "cov_std": 0.4068885110318661, |
| "entropy": 0.4091796875, |
| "epoch": 0.11085714285714286, |
| "grad_norm": 0.28435367345809937, |
| "kl": 0.0007953643798828125, |
| "learning_rate": 1.0246514708427701e-07, |
| "loss": -0.0236, |
| "reward": 0.5208333563059568, |
| "reward_std": 0.5388510599732399, |
| "rewards/accuracy_reward": 0.19791667722165585, |
| "rewards/format_reward": 0.3229166818782687, |
| "step": 97, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04683105740696192, |
| "w_max": 1.4916425049304962, |
| "w_mean": 1.1599352061748505, |
| "w_min": 0.0, |
| "w_std": 0.24916893057525158 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3058.0209350585938, |
| "cov_mean": 0.00017139434021373745, |
| "cov_std": 0.42342133820056915, |
| "entropy": 0.37744140625, |
| "epoch": 0.112, |
| "grad_norm": 0.39048805832862854, |
| "kl": 0.0005214214324951172, |
| "learning_rate": 1.0109617738307911e-07, |
| "loss": -0.044, |
| "reward": 0.645833358168602, |
| "reward_std": 0.4053402505815029, |
| "rewards/accuracy_reward": 0.1770833432674408, |
| "rewards/format_reward": 0.4687500149011612, |
| "step": 98, |
| "w_high_ratio": 0.04422338306903839, |
| "w_low_ratio": 0.044881118927150965, |
| "w_max": 1.805379331111908, |
| "w_mean": 1.279258906841278, |
| "w_min": 9.581378249820937e-43, |
| "w_std": 0.26856033504009247 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 3090.6771545410156, |
| "cov_mean": -1.8359070736551075e-05, |
| "cov_std": 0.19972562417387962, |
| "entropy": 0.34521484375, |
| "epoch": 0.11314285714285714, |
| "grad_norm": 0.27053090929985046, |
| "kl": 0.0011525154113769531, |
| "learning_rate": 1.002741278414069e-07, |
| "loss": 0.0295, |
| "reward": 0.5416666893288493, |
| "reward_std": 0.29046234488487244, |
| "rewards/accuracy_reward": 0.2187500074505806, |
| "rewards/format_reward": 0.3229166669771075, |
| "step": 99, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.028498666360974312, |
| "w_max": 1.280459076166153, |
| "w_mean": 1.0813924968242645, |
| "w_min": 0.25, |
| "w_std": 0.1357164867222309 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 2991.7500610351562, |
| "cov_mean": -9.51583083406149e-05, |
| "cov_std": 0.3894283100962639, |
| "entropy": 0.356201171875, |
| "epoch": 0.11428571428571428, |
| "grad_norm": 0.5418879389762878, |
| "kl": 0.00218963623046875, |
| "learning_rate": 1e-07, |
| "loss": -0.0626, |
| "reward": 0.7395833432674408, |
| "reward_std": 0.479395255446434, |
| "rewards/accuracy_reward": 0.2916666716337204, |
| "rewards/format_reward": 0.4479166865348816, |
| "step": 100, |
| "w_high_ratio": 0.14099212735891342, |
| "w_low_ratio": 0.036807731725275517, |
| "w_max": 2.043416380882263, |
| "w_mean": 1.3361191749572754, |
| "w_min": 6.562710904496729e-32, |
| "w_std": 0.25866811722517014 |
| }, |
| { |
| "epoch": 0.11428571428571428, |
| "step": 100, |
| "total_flos": 0.0, |
| "train_loss": -0.004950485862791538, |
| "train_runtime": 8385.5538, |
| "train_samples_per_second": 1.145, |
| "train_steps_per_second": 0.012 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 100, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 6, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
