{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.0105, "eval_steps": 500, "global_step": 525, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 18.125, "completions/mean_terminated_length": 12.733333587646484, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9520235098898411, "epoch": 2e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.55356764793396, "kl": 0.0, "learning_rate": 0.0, "loss": -0.1852, "num_tokens": 44763.0, "reward": -0.28437501192092896, "reward_std": 0.29535973072052, "rewards/rollout_reward_func/mean": -0.28437501192092896, "rewards/rollout_reward_func/std": 0.2963317632675171, "sampling/importance_sampling_ratio/max": 1.2497165203094482, "sampling/importance_sampling_ratio/mean": 0.6355467438697815, "sampling/importance_sampling_ratio/min": 1.1565955989533244e-38, "sampling/sampling_logp_difference/max": 32.93802261352539, "sampling/sampling_logp_difference/mean": 1.07814621925354, "step": 1, "step_time": 18.965608670012443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.5, "completions/mean_terminated_length": 16.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9043922126293182, "epoch": 4e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.4334965944290161, "kl": 0.0, "learning_rate": 2.8571428571428575e-07, "loss": -0.1987, "num_tokens": 90357.0, "reward": -0.3062500059604645, "reward_std": 0.25588592886924744, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.2601953446865082, "sampling/importance_sampling_ratio/max": 1.3774664402008057, "sampling/importance_sampling_ratio/mean": 0.7802153825759888, "sampling/importance_sampling_ratio/min": 8.88554343076644e-37, "sampling/sampling_logp_difference/max": 34.01368713378906, "sampling/sampling_logp_difference/mean": 0.5977065563201904, "step": 2, "step_time": 19.190262573974906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.03125, "completions/mean_terminated_length": 14.0625, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9372651912271976, "epoch": 6e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.45659568905830383, "kl": 0.0007417532597173704, "learning_rate": 5.714285714285715e-07, "loss": -0.0451, "num_tokens": 133984.0, "reward": -0.2718750238418579, "reward_std": 0.22434553503990173, "rewards/rollout_reward_func/mean": -0.2718750238418579, "rewards/rollout_reward_func/std": 0.25429773330688477, "sampling/importance_sampling_ratio/max": 1.257419228553772, "sampling/importance_sampling_ratio/mean": 0.7342961430549622, "sampling/importance_sampling_ratio/min": 5.267357613132127e-39, "sampling/sampling_logp_difference/max": 34.34855270385742, "sampling/sampling_logp_difference/mean": 0.6541591286659241, "step": 3, "step_time": 18.255347436948796 }, { "clip_ratio/high_max": 0.004807692486792803, "clip_ratio/high_mean": 0.0024038462433964014, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024038462433964014, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 16.1875, "completions/mean_terminated_length": 14.285714149475098, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.983771501109004, "epoch": 8e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.3951721787452698, "kl": 0.0008293715673062252, "learning_rate": 8.571428571428572e-07, "loss": 0.0478, "num_tokens": 178644.0, "reward": -0.30000001192092896, "reward_std": 0.1853259801864624, "rewards/rollout_reward_func/mean": -0.30000001192092896, "rewards/rollout_reward_func/std": 0.1866512894630432, "sampling/importance_sampling_ratio/max": 1.411834955215454, "sampling/importance_sampling_ratio/mean": 0.7878267168998718, "sampling/importance_sampling_ratio/min": 6.095648319812954e-43, "sampling/sampling_logp_difference/max": 32.21796798706055, "sampling/sampling_logp_difference/mean": 0.7324473857879639, "step": 4, "step_time": 18.257113829022273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.9375, "completions/mean_terminated_length": 15.800000190734863, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0402559489011765, "epoch": 0.0001, "frac_reward_zero_std": 0.0, "grad_norm": 0.48684850335121155, "kl": 0.0009579719044268131, "learning_rate": 1.142857142857143e-06, "loss": -0.1786, "num_tokens": 223489.0, "reward": -0.3062500059604645, "reward_std": 0.3002167046070099, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.31205615401268005, "sampling/importance_sampling_ratio/max": 1.4054697751998901, "sampling/importance_sampling_ratio/mean": 0.627733051776886, "sampling/importance_sampling_ratio/min": 9.339433195879174e-37, "sampling/sampling_logp_difference/max": 34.00166702270508, "sampling/sampling_logp_difference/mean": 1.009493112564087, "step": 5, "step_time": 20.06307427198044 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 15.59375, "completions/mean_terminated_length": 12.65217399597168, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8666270300745964, "epoch": 0.00012, "frac_reward_zero_std": 0.0, "grad_norm": 0.26274776458740234, "kl": 0.0007614124756400997, "learning_rate": 1.4285714285714286e-06, "loss": -0.0541, "num_tokens": 268340.0, "reward": -0.3375000059604645, "reward_std": 0.15463024377822876, "rewards/rollout_reward_func/mean": -0.3375000059604645, "rewards/rollout_reward_func/std": 0.16214290261268616, "sampling/importance_sampling_ratio/max": 1.385573148727417, "sampling/importance_sampling_ratio/mean": 0.7455233335494995, "sampling/importance_sampling_ratio/min": 3.313567835610663e-36, "sampling/sampling_logp_difference/max": 32.64977264404297, "sampling/sampling_logp_difference/mean": 0.6757885217666626, "step": 6, "step_time": 17.820020423983806 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023593305377289653, "completions/clipped_ratio": 0.5, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 16.3125, "completions/mean_terminated_length": 14.625, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9107956029474735, "epoch": 0.00014, "frac_reward_zero_std": 0.0, "grad_norm": 0.4873211979866028, "kl": 0.0006822368086432107, "learning_rate": 1.7142857142857145e-06, "loss": -0.0341, "num_tokens": 312768.0, "reward": -0.26250001788139343, "reward_std": 0.2416815459728241, "rewards/rollout_reward_func/mean": -0.26250001788139343, "rewards/rollout_reward_func/std": 0.2612006962299347, "sampling/importance_sampling_ratio/max": 1.178691029548645, "sampling/importance_sampling_ratio/mean": 0.7348640561103821, "sampling/importance_sampling_ratio/min": 9.690568089783363e-29, "sampling/sampling_logp_difference/max": 32.261634826660156, "sampling/sampling_logp_difference/mean": 0.6254305839538574, "step": 7, "step_time": 19.314552246010862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 16.625, "completions/mean_terminated_length": 13.476190567016602, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9005667716264725, "epoch": 0.00016, "frac_reward_zero_std": 0.0, "grad_norm": 0.33328598737716675, "kl": 0.000907028097572038, "learning_rate": 2.0000000000000003e-06, "loss": 0.0483, "num_tokens": 357959.0, "reward": -0.2750000059604645, "reward_std": 0.2331797480583191, "rewards/rollout_reward_func/mean": -0.2750000059604645, "rewards/rollout_reward_func/std": 0.24362848699092865, "sampling/importance_sampling_ratio/max": 1.248006820678711, "sampling/importance_sampling_ratio/mean": 0.6755537390708923, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.056026458740234, "sampling/sampling_logp_difference/mean": 1.0328199863433838, "step": 8, "step_time": 18.56200566799089 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0010080644860863686, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010080644860863686, "completions/clipped_ratio": 0.5, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.09375, "completions/mean_terminated_length": 15.1875, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9855761677026749, "epoch": 0.00018, "frac_reward_zero_std": 0.0, "grad_norm": 0.5635325908660889, "kl": 0.0008607167783338809, "learning_rate": 2.285714285714286e-06, "loss": -0.1201, "num_tokens": 401174.0, "reward": -0.26875001192092896, "reward_std": 0.26515159010887146, "rewards/rollout_reward_func/mean": -0.26875001192092896, "rewards/rollout_reward_func/std": 0.27990493178367615, "sampling/importance_sampling_ratio/max": 1.2465877532958984, "sampling/importance_sampling_ratio/mean": 0.7913148999214172, "sampling/importance_sampling_ratio/min": 2.449086517682765e-35, "sampling/sampling_logp_difference/max": 32.81730651855469, "sampling/sampling_logp_difference/mean": 0.5978679656982422, "step": 9, "step_time": 18.987721519966726 }, { "clip_ratio/high_max": 0.0069444444961845875, "clip_ratio/high_mean": 0.0034722222480922937, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034722222480922937, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 14.3125, "completions/mean_terminated_length": 12.090909004211426, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8981539234519005, "epoch": 0.0002, "frac_reward_zero_std": 0.0, "grad_norm": 0.6798989176750183, "kl": 0.0007426400807162281, "learning_rate": 2.571428571428571e-06, "loss": 0.038, "num_tokens": 445117.0, "reward": -0.25, "reward_std": 0.18156909942626953, "rewards/rollout_reward_func/mean": -0.25, "rewards/rollout_reward_func/std": 0.21701790392398834, "sampling/importance_sampling_ratio/max": 1.3405351638793945, "sampling/importance_sampling_ratio/mean": 0.8464469909667969, "sampling/importance_sampling_ratio/min": 7.780345385562822e-40, "sampling/sampling_logp_difference/max": 33.585845947265625, "sampling/sampling_logp_difference/mean": 0.5090183019638062, "step": 10, "step_time": 19.219402696006 }, { "clip_ratio/high_max": 0.004032257944345474, "clip_ratio/high_mean": 0.002016128972172737, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002016128972172737, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.5, "completions/mean_terminated_length": 15.333333969116211, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9444097988307476, "epoch": 0.00022, "frac_reward_zero_std": 0.0, "grad_norm": 0.4182659387588501, "kl": 0.0012234995820108452, "learning_rate": 2.8571428571428573e-06, "loss": -0.0608, "num_tokens": 490162.0, "reward": -0.23750001192092896, "reward_std": 0.28101080656051636, "rewards/rollout_reward_func/mean": -0.23750001192092896, "rewards/rollout_reward_func/std": 0.2981123626232147, "sampling/importance_sampling_ratio/max": 1.163366675376892, "sampling/importance_sampling_ratio/mean": 0.6333475112915039, "sampling/importance_sampling_ratio/min": 4.766586903427959e-37, "sampling/sampling_logp_difference/max": 33.50480270385742, "sampling/sampling_logp_difference/mean": 0.8179425001144409, "step": 11, "step_time": 18.349101219006116 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0010775862028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030307112028822303, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 15.25, "completions/mean_terminated_length": 12.30434799194336, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9315127320587635, "epoch": 0.00024, "frac_reward_zero_std": 0.0, "grad_norm": 0.4497344195842743, "kl": 0.0010870849018829176, "learning_rate": 3.142857142857143e-06, "loss": -0.1762, "num_tokens": 533795.0, "reward": -0.3187500238418579, "reward_std": 0.169805645942688, "rewards/rollout_reward_func/mean": -0.3187500238418579, "rewards/rollout_reward_func/std": 0.18567661941051483, "sampling/importance_sampling_ratio/max": 1.4432153701782227, "sampling/importance_sampling_ratio/mean": 0.7977529764175415, "sampling/importance_sampling_ratio/min": 5.021706180804163e-37, "sampling/sampling_logp_difference/max": 33.26485824584961, "sampling/sampling_logp_difference/mean": 0.7735878229141235, "step": 12, "step_time": 18.737031336015207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 21.5, "completions/mean_terminated_length": 18.285715103149414, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0399756655097008, "epoch": 0.00026, "frac_reward_zero_std": 0.0, "grad_norm": 0.3983003795146942, "kl": 0.0007745381099084625, "learning_rate": 3.428571428571429e-06, "loss": -0.2039, "num_tokens": 579162.0, "reward": -0.34375, "reward_std": 0.2869546711444855, "rewards/rollout_reward_func/mean": -0.34375, "rewards/rollout_reward_func/std": 0.29723185300827026, "sampling/importance_sampling_ratio/max": 1.1047006845474243, "sampling/importance_sampling_ratio/mean": 0.47800928354263306, "sampling/importance_sampling_ratio/min": 1.2578055015779558e-41, "sampling/sampling_logp_difference/max": 32.942298889160156, "sampling/sampling_logp_difference/mean": 0.9449502825737, "step": 13, "step_time": 19.463258072006283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0014204545877873898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014204545877873898, "completions/clipped_ratio": 0.4375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.625, "completions/mean_terminated_length": 16.38888931274414, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9908140040934086, "epoch": 0.00028, "frac_reward_zero_std": 0.0, "grad_norm": 0.2862341105937958, "kl": 0.0008231814663304249, "learning_rate": 3.7142857142857146e-06, "loss": -0.0487, "num_tokens": 622533.0, "reward": -0.34062501788139343, "reward_std": 0.17591193318367004, "rewards/rollout_reward_func/mean": -0.34062501788139343, "rewards/rollout_reward_func/std": 0.17754486203193665, "sampling/importance_sampling_ratio/max": 1.1499879360198975, "sampling/importance_sampling_ratio/mean": 0.6184262037277222, "sampling/importance_sampling_ratio/min": 1.581197161174837e-40, "sampling/sampling_logp_difference/max": 33.62101364135742, "sampling/sampling_logp_difference/mean": 1.0848355293273926, "step": 14, "step_time": 19.85021109401714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 16.78125, "completions/mean_terminated_length": 14.722222328186035, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9377644173800945, "epoch": 0.0003, "frac_reward_zero_std": 0.0, "grad_norm": 0.6352251768112183, "kl": 0.0009140272268268745, "learning_rate": 4.000000000000001e-06, "loss": 0.021, "num_tokens": 666394.0, "reward": -0.22812500596046448, "reward_std": 0.2589312195777893, "rewards/rollout_reward_func/mean": -0.22812500596046448, "rewards/rollout_reward_func/std": 0.2819681763648987, "sampling/importance_sampling_ratio/max": 1.2279274463653564, "sampling/importance_sampling_ratio/mean": 0.8032234907150269, "sampling/importance_sampling_ratio/min": 2.9930764828391135e-23, "sampling/sampling_logp_difference/max": 23.261566162109375, "sampling/sampling_logp_difference/mean": 0.5142928957939148, "step": 15, "step_time": 18.171195832954254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0014204545877873898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014204545877873898, "completions/clipped_ratio": 0.5, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.375, "completions/mean_terminated_length": 15.75, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0404513776302338, "epoch": 0.00032, "frac_reward_zero_std": 0.0, "grad_norm": 0.5324226021766663, "kl": 0.0010930900898529217, "learning_rate": 4.2857142857142855e-06, "loss": 0.0645, "num_tokens": 711657.0, "reward": -0.29375001788139343, "reward_std": 0.20063459873199463, "rewards/rollout_reward_func/mean": -0.29375001788139343, "rewards/rollout_reward_func/std": 0.21089287102222443, "sampling/importance_sampling_ratio/max": 1.342148780822754, "sampling/importance_sampling_ratio/mean": 0.7169525027275085, "sampling/importance_sampling_ratio/min": 4.066790949926447e-39, "sampling/sampling_logp_difference/max": 33.52042770385742, "sampling/sampling_logp_difference/mean": 0.6504959464073181, "step": 16, "step_time": 18.525358717975905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 17.8125, "completions/mean_terminated_length": 15.333333969116211, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.040733888745308, "epoch": 0.00034, "frac_reward_zero_std": 0.0, "grad_norm": 0.5637218356132507, "kl": 0.0018714752222876996, "learning_rate": 4.571428571428572e-06, "loss": -0.1254, "num_tokens": 756303.0, "reward": -0.22187501192092896, "reward_std": 0.3509283661842346, "rewards/rollout_reward_func/mean": -0.22187501192092896, "rewards/rollout_reward_func/std": 0.3414715528488159, "sampling/importance_sampling_ratio/max": 1.1514338254928589, "sampling/importance_sampling_ratio/mean": 0.5738069415092468, "sampling/importance_sampling_ratio/min": 2.0879347118439774e-43, "sampling/sampling_logp_difference/max": 33.327354431152344, "sampling/sampling_logp_difference/mean": 1.080234169960022, "step": 17, "step_time": 20.696848441002658 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012019231216982007, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 17.78125, "completions/mean_terminated_length": 16.571428298950195, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0740827918052673, "epoch": 0.00036, "frac_reward_zero_std": 0.0, "grad_norm": 0.3453066349029541, "kl": 0.0011216673974558944, "learning_rate": 4.857142857142858e-06, "loss": -0.0236, "num_tokens": 801655.0, "reward": -0.234375, "reward_std": 0.23094896972179413, "rewards/rollout_reward_func/mean": -0.234375, "rewards/rollout_reward_func/std": 0.2522279620170593, "sampling/importance_sampling_ratio/max": 1.2593371868133545, "sampling/importance_sampling_ratio/mean": 0.6604970693588257, "sampling/importance_sampling_ratio/min": 2.4241482523894308e-40, "sampling/sampling_logp_difference/max": 33.579830169677734, "sampling/sampling_logp_difference/mean": 1.06025230884552, "step": 18, "step_time": 18.792985362932086 }, { "clip_ratio/high_max": 0.002016128972172737, "clip_ratio/high_mean": 0.0010080644860863686, "clip_ratio/low_mean": 0.0009765625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019846269860863686, "completions/clipped_ratio": 0.5, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.875, "completions/mean_terminated_length": 13.75, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9512934274971485, "epoch": 0.00038, "frac_reward_zero_std": 0.0, "grad_norm": 0.36085009574890137, "kl": 0.0017993518049479462, "learning_rate": 5.142857142857142e-06, "loss": 0.0104, "num_tokens": 846550.0, "reward": -0.24687501788139343, "reward_std": 0.2874893844127655, "rewards/rollout_reward_func/mean": -0.24687501788139343, "rewards/rollout_reward_func/std": 0.2929101884365082, "sampling/importance_sampling_ratio/max": 1.5822679996490479, "sampling/importance_sampling_ratio/mean": 0.6975961327552795, "sampling/importance_sampling_ratio/min": 8.225621985586676e-43, "sampling/sampling_logp_difference/max": 33.42110061645508, "sampling/sampling_logp_difference/mean": 0.8275338411331177, "step": 19, "step_time": 19.412421728033223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0040990260895341635, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0040990260895341635, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.6875, "completions/mean_terminated_length": 14.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9852762538939714, "epoch": 0.0004, "frac_reward_zero_std": 0.0, "grad_norm": 0.5814059972763062, "kl": 0.004399896068207454, "learning_rate": 5.428571428571429e-06, "loss": -0.0863, "num_tokens": 891101.0, "reward": -0.265625, "reward_std": 0.3245283365249634, "rewards/rollout_reward_func/mean": -0.265625, "rewards/rollout_reward_func/std": 0.32389700412750244, "sampling/importance_sampling_ratio/max": 1.4829518795013428, "sampling/importance_sampling_ratio/mean": 0.6164427995681763, "sampling/importance_sampling_ratio/min": 1.485596656302898e-38, "sampling/sampling_logp_difference/max": 33.387168884277344, "sampling/sampling_logp_difference/mean": 1.0106862783432007, "step": 20, "step_time": 18.477307860026485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.46875, "completions/max_length": 32.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 17.8125, "completions/mean_terminated_length": 12.823529243469238, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9481113106012344, "epoch": 0.00042, "frac_reward_zero_std": 0.0, "grad_norm": 0.5848221778869629, "kl": 0.005494226890732534, "learning_rate": 5.7142857142857145e-06, "loss": -0.1149, "num_tokens": 935601.0, "reward": -0.24062500894069672, "reward_std": 0.33860644698143005, "rewards/rollout_reward_func/mean": -0.24062500894069672, "rewards/rollout_reward_func/std": 0.34720948338508606, "sampling/importance_sampling_ratio/max": 1.2147438526153564, "sampling/importance_sampling_ratio/mean": 0.6370307207107544, "sampling/importance_sampling_ratio/min": 1.9594078730353777e-29, "sampling/sampling_logp_difference/max": 31.31047821044922, "sampling/sampling_logp_difference/mean": 0.9164257645606995, "step": 21, "step_time": 19.010053597958176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 16.71875, "completions/mean_terminated_length": 13.1578950881958, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.868434477597475, "epoch": 0.00044, "frac_reward_zero_std": 0.0, "grad_norm": 0.4228771924972534, "kl": 0.003879194518958684, "learning_rate": 6e-06, "loss": -0.04, "num_tokens": 980620.0, "reward": -0.2406250238418579, "reward_std": 0.2776505947113037, "rewards/rollout_reward_func/mean": -0.2406250238418579, "rewards/rollout_reward_func/std": 0.2872105836868286, "sampling/importance_sampling_ratio/max": 1.307322382926941, "sampling/importance_sampling_ratio/mean": 0.7323261499404907, "sampling/importance_sampling_ratio/min": 1.747419185013047e-42, "sampling/sampling_logp_difference/max": 33.80165481567383, "sampling/sampling_logp_difference/mean": 0.8035021424293518, "step": 22, "step_time": 19.73815098000341 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.46875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.1875, "completions/mean_terminated_length": 16.52941131591797, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8888803906738758, "epoch": 0.00046, "frac_reward_zero_std": 0.0, "grad_norm": 0.5111181139945984, "kl": 0.006473908113548532, "learning_rate": 6.285714285714286e-06, "loss": -0.2474, "num_tokens": 1025684.0, "reward": -0.33125001192092896, "reward_std": 0.2876274883747101, "rewards/rollout_reward_func/mean": -0.33125001192092896, "rewards/rollout_reward_func/std": 0.28897762298583984, "sampling/importance_sampling_ratio/max": 1.2532511949539185, "sampling/importance_sampling_ratio/mean": 0.6077173948287964, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.46676254272461, "sampling/sampling_logp_difference/mean": 0.9781787395477295, "step": 23, "step_time": 19.071067536016926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 18.5, "completions/mean_terminated_length": 16.947368621826172, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.9010880440473557, "epoch": 0.00048, "frac_reward_zero_std": 0.0, "grad_norm": 0.3238581418991089, "kl": 0.0033895968153956346, "learning_rate": 6.571428571428572e-06, "loss": 0.0121, "num_tokens": 1070497.0, "reward": -0.2718750238418579, "reward_std": 0.21975678205490112, "rewards/rollout_reward_func/mean": -0.2718750238418579, "rewards/rollout_reward_func/std": 0.22031410038471222, "sampling/importance_sampling_ratio/max": 1.4573770761489868, "sampling/importance_sampling_ratio/mean": 0.7308536767959595, "sampling/importance_sampling_ratio/min": 3.346637044439101e-40, "sampling/sampling_logp_difference/max": 33.01390075683594, "sampling/sampling_logp_difference/mean": 0.7329796552658081, "step": 24, "step_time": 20.29044062299363 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0031105324160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004267939832061529, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 15.78125, "completions/mean_terminated_length": 15.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.925279650837183, "epoch": 0.0005, "frac_reward_zero_std": 0.0, "grad_norm": 0.8869653940200806, "kl": 0.009519676546915434, "learning_rate": 6.857142857142858e-06, "loss": 0.0285, "num_tokens": 1113994.0, "reward": -0.15000000596046448, "reward_std": 0.27477729320526123, "rewards/rollout_reward_func/mean": -0.15000000596046448, "rewards/rollout_reward_func/std": 0.285114586353302, "sampling/importance_sampling_ratio/max": 1.5002343654632568, "sampling/importance_sampling_ratio/mean": 0.8037615418434143, "sampling/importance_sampling_ratio/min": 2.9273965698824024e-40, "sampling/sampling_logp_difference/max": 33.992557525634766, "sampling/sampling_logp_difference/mean": 0.8297852277755737, "step": 25, "step_time": 18.252676502001123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.65625, "completions/mean_terminated_length": 18.40909194946289, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9806470051407814, "epoch": 0.00052, "frac_reward_zero_std": 0.0, "grad_norm": 0.25246933102607727, "kl": 0.007313871057704091, "learning_rate": 7.1428571428571436e-06, "loss": -0.0113, "num_tokens": 1159629.0, "reward": -0.2906250059604645, "reward_std": 0.23580847680568695, "rewards/rollout_reward_func/mean": -0.2906250059604645, "rewards/rollout_reward_func/std": 0.2346711903810501, "sampling/importance_sampling_ratio/max": 1.4894014596939087, "sampling/importance_sampling_ratio/mean": 0.5861202478408813, "sampling/importance_sampling_ratio/min": 2.3261554507791963e-43, "sampling/sampling_logp_difference/max": 33.06056213378906, "sampling/sampling_logp_difference/mean": 0.8995763063430786, "step": 26, "step_time": 19.457321609996143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011574074160307646, "completions/clipped_ratio": 0.40625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.375, "completions/mean_terminated_length": 16.63157844543457, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0120329521596432, "epoch": 0.00054, "frac_reward_zero_std": 0.0, "grad_norm": 0.539905309677124, "kl": 0.011712428560713306, "learning_rate": 7.428571428571429e-06, "loss": -0.117, "num_tokens": 1203494.0, "reward": -0.2718749940395355, "reward_std": 0.27511081099510193, "rewards/rollout_reward_func/mean": -0.2718749940395355, "rewards/rollout_reward_func/std": 0.26425108313560486, "sampling/importance_sampling_ratio/max": 1.3162537813186646, "sampling/importance_sampling_ratio/mean": 0.7684336304664612, "sampling/importance_sampling_ratio/min": 7.774403880074085e-42, "sampling/sampling_logp_difference/max": 34.02478790283203, "sampling/sampling_logp_difference/mean": 0.8027687072753906, "step": 27, "step_time": 20.147836231000838 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00930059514939785, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 15.5625, "completions/mean_terminated_length": 14.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8481537066400051, "epoch": 0.00056, "frac_reward_zero_std": 0.0, "grad_norm": 0.5762486457824707, "kl": 0.04291028264560737, "learning_rate": 7.714285714285716e-06, "loss": 0.024, "num_tokens": 1246796.0, "reward": -0.125, "reward_std": 0.30612248182296753, "rewards/rollout_reward_func/mean": -0.125, "rewards/rollout_reward_func/std": 0.3005371689796448, "sampling/importance_sampling_ratio/max": 1.3473433256149292, "sampling/importance_sampling_ratio/mean": 0.7729018330574036, "sampling/importance_sampling_ratio/min": 1.2405288928112951e-39, "sampling/sampling_logp_difference/max": 33.277793884277344, "sampling/sampling_logp_difference/mean": 0.7352918982505798, "step": 28, "step_time": 17.947007385941106 }, { "clip_ratio/high_max": 0.001953125, "clip_ratio/high_mean": 0.0009765625, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029296875, "completions/clipped_ratio": 0.375, "completions/max_length": 32.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 17.0625, "completions/mean_terminated_length": 13.699999809265137, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9322966746985912, "epoch": 0.00058, "frac_reward_zero_std": 0.0, "grad_norm": 0.3211604356765747, "kl": 0.06583275100274477, "learning_rate": 8.000000000000001e-06, "loss": -0.0652, "num_tokens": 1292034.0, "reward": -0.18437501788139343, "reward_std": 0.23132722079753876, "rewards/rollout_reward_func/mean": -0.18437501788139343, "rewards/rollout_reward_func/std": 0.2908379137516022, "sampling/importance_sampling_ratio/max": 1.4799343347549438, "sampling/importance_sampling_ratio/mean": 0.6418001651763916, "sampling/importance_sampling_ratio/min": 4.4719637891997887e-41, "sampling/sampling_logp_difference/max": 33.431026458740234, "sampling/sampling_logp_difference/mean": 1.141252875328064, "step": 29, "step_time": 19.658783023012802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 14.53125, "completions/mean_terminated_length": 13.576923370361328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7947591431438923, "epoch": 0.0006, "frac_reward_zero_std": 0.0, "grad_norm": 0.4960869252681732, "kl": 0.05044753081165254, "learning_rate": 8.285714285714287e-06, "loss": -0.0798, "num_tokens": 1336974.0, "reward": -0.20937499403953552, "reward_std": 0.2394784688949585, "rewards/rollout_reward_func/mean": -0.20937499403953552, "rewards/rollout_reward_func/std": 0.2656208276748657, "sampling/importance_sampling_ratio/max": 1.3650352954864502, "sampling/importance_sampling_ratio/mean": 0.7317414283752441, "sampling/importance_sampling_ratio/min": 3.549922011360238e-38, "sampling/sampling_logp_difference/max": 31.6182804107666, "sampling/sampling_logp_difference/mean": 0.7568868398666382, "step": 30, "step_time": 17.777866849006386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.125, "completions/mean_terminated_length": 14.88888931274414, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9024674706161022, "epoch": 0.00062, "frac_reward_zero_std": 0.0, "grad_norm": 0.5782524943351746, "kl": 0.08034574758494273, "learning_rate": 8.571428571428571e-06, "loss": -0.2472, "num_tokens": 1381837.0, "reward": -0.24375000596046448, "reward_std": 0.2614990472793579, "rewards/rollout_reward_func/mean": -0.24375000596046448, "rewards/rollout_reward_func/std": 0.2906472086906433, "sampling/importance_sampling_ratio/max": 1.485732078552246, "sampling/importance_sampling_ratio/mean": 0.5952891111373901, "sampling/importance_sampling_ratio/min": 1.5355008182532048e-40, "sampling/sampling_logp_difference/max": 33.827354431152344, "sampling/sampling_logp_difference/mean": 1.0518423318862915, "step": 31, "step_time": 19.294928691000678 }, { "clip_ratio/high_max": 0.0020833334419876337, "clip_ratio/high_mean": 0.0010416667209938169, "clip_ratio/low_mean": 0.0010775862028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002119252923876047, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 16.125, "completions/mean_terminated_length": 14.86363697052002, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9107420369982719, "epoch": 0.00064, "frac_reward_zero_std": 0.0, "grad_norm": 0.4850414991378784, "kl": 0.04685692639031913, "learning_rate": 8.857142857142858e-06, "loss": 0.0035, "num_tokens": 1427501.0, "reward": -0.13750000298023224, "reward_std": 0.27463147044181824, "rewards/rollout_reward_func/mean": -0.13750000298023224, "rewards/rollout_reward_func/std": 0.266094833612442, "sampling/importance_sampling_ratio/max": 1.389861822128296, "sampling/importance_sampling_ratio/mean": 0.8197333812713623, "sampling/importance_sampling_ratio/min": 5.584174380334396e-42, "sampling/sampling_logp_difference/max": 33.66105270385742, "sampling/sampling_logp_difference/mean": 1.154968023300171, "step": 32, "step_time": 18.852268524977262 }, { "clip_ratio/high_max": 0.007068452658131719, "clip_ratio/high_mean": 0.0035342263290658593, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035342263290658593, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 15.6875, "completions/mean_terminated_length": 14.960000038146973, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7761146072298288, "epoch": 0.00066, "frac_reward_zero_std": 0.0, "grad_norm": 0.4796203374862671, "kl": 0.07175984373316169, "learning_rate": 9.142857142857144e-06, "loss": 0.0098, "num_tokens": 1471606.0, "reward": -0.09062500298023224, "reward_std": 0.28281545639038086, "rewards/rollout_reward_func/mean": -0.09062500298023224, "rewards/rollout_reward_func/std": 0.29111507534980774, "sampling/importance_sampling_ratio/max": 1.612539291381836, "sampling/importance_sampling_ratio/mean": 0.8781986236572266, "sampling/importance_sampling_ratio/min": 1.1210387714598537e-44, "sampling/sampling_logp_difference/max": 33.4826545715332, "sampling/sampling_logp_difference/mean": 0.7881132364273071, "step": 33, "step_time": 17.811717419011984 }, { "clip_ratio/high_max": 0.0020833334419876337, "clip_ratio/high_mean": 0.0010416667209938169, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024003623984754086, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 17.65625, "completions/mean_terminated_length": 14.09523868560791, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7851596139371395, "epoch": 0.00068, "frac_reward_zero_std": 0.0, "grad_norm": 0.5333690643310547, "kl": 0.14519566847593524, "learning_rate": 9.42857142857143e-06, "loss": -0.2126, "num_tokens": 1516648.0, "reward": -0.21562501788139343, "reward_std": 0.3383692502975464, "rewards/rollout_reward_func/mean": -0.21562501788139343, "rewards/rollout_reward_func/std": 0.3273642361164093, "sampling/importance_sampling_ratio/max": 1.4924228191375732, "sampling/importance_sampling_ratio/mean": 0.7424999475479126, "sampling/importance_sampling_ratio/min": 8.036446692902826e-40, "sampling/sampling_logp_difference/max": 33.21634292602539, "sampling/sampling_logp_difference/mean": 0.7939122915267944, "step": 34, "step_time": 19.001740949985106 }, { "clip_ratio/high_max": 0.005073052132502198, "clip_ratio/high_mean": 0.002536526066251099, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004768669023178518, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 15.71875, "completions/mean_terminated_length": 14.481481552124023, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.020354587584734, "epoch": 0.0007, "frac_reward_zero_std": 0.0, "grad_norm": 0.36148887872695923, "kl": 0.08580824709497392, "learning_rate": 9.714285714285715e-06, "loss": -0.0401, "num_tokens": 1562234.0, "reward": -0.16875000298023224, "reward_std": 0.2685546278953552, "rewards/rollout_reward_func/mean": -0.16875000298023224, "rewards/rollout_reward_func/std": 0.2810550332069397, "sampling/importance_sampling_ratio/max": 1.6110713481903076, "sampling/importance_sampling_ratio/mean": 0.7410909533500671, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.713829040527344, "sampling/sampling_logp_difference/mean": 1.4497594833374023, "step": 35, "step_time": 17.894258211032138 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0031250000465661287, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007031250046566129, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 14.0625, "completions/mean_terminated_length": 12.239999771118164, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.6617361567914486, "epoch": 0.00072, "frac_reward_zero_std": 0.0, "grad_norm": 0.6173807382583618, "kl": 0.3624987488728948, "learning_rate": 1e-05, "loss": -0.2383, "num_tokens": 1606454.0, "reward": -0.2187500149011612, "reward_std": 0.2674262821674347, "rewards/rollout_reward_func/mean": -0.2187500149011612, "rewards/rollout_reward_func/std": 0.2822004556655884, "sampling/importance_sampling_ratio/max": 1.5456047058105469, "sampling/importance_sampling_ratio/mean": 0.7621349096298218, "sampling/importance_sampling_ratio/min": 3.2482653552977456e-27, "sampling/sampling_logp_difference/max": 17.206562042236328, "sampling/sampling_logp_difference/mean": 0.5444260835647583, "step": 36, "step_time": 17.77500358998077 }, { "clip_ratio/high_max": 0.007031250046566129, "clip_ratio/high_mean": 0.0035156250232830644, "clip_ratio/low_mean": 0.0010775862028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0045932113425806165, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 16.9375, "completions/mean_terminated_length": 16.045455932617188, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7834046334028244, "epoch": 0.00074, "frac_reward_zero_std": 0.0, "grad_norm": 0.5569875240325928, "kl": 0.11322040017694235, "learning_rate": 9.999975267482496e-06, "loss": -0.0055, "num_tokens": 1651491.0, "reward": -0.16562500596046448, "reward_std": 0.2949869632720947, "rewards/rollout_reward_func/mean": -0.16562500596046448, "rewards/rollout_reward_func/std": 0.28805169463157654, "sampling/importance_sampling_ratio/max": 1.484552025794983, "sampling/importance_sampling_ratio/mean": 0.7655913829803467, "sampling/importance_sampling_ratio/min": 5.324934164434305e-44, "sampling/sampling_logp_difference/max": 32.688995361328125, "sampling/sampling_logp_difference/mean": 0.7902384996414185, "step": 37, "step_time": 18.1823700209643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 19.6875, "completions/mean_terminated_length": 16.3157901763916, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9248369373381138, "epoch": 0.00076, "frac_reward_zero_std": 0.0, "grad_norm": 0.31241974234580994, "kl": 0.13545625447295606, "learning_rate": 9.99990107025622e-06, "loss": -0.1645, "num_tokens": 1696986.0, "reward": -0.21250000596046448, "reward_std": 0.29384416341781616, "rewards/rollout_reward_func/mean": -0.21250000596046448, "rewards/rollout_reward_func/std": 0.3034745156764984, "sampling/importance_sampling_ratio/max": 1.5557458400726318, "sampling/importance_sampling_ratio/mean": 0.6030323505401611, "sampling/importance_sampling_ratio/min": 7.42688186092153e-44, "sampling/sampling_logp_difference/max": 33.452354431152344, "sampling/sampling_logp_difference/mean": 1.0540035963058472, "step": 38, "step_time": 18.306734503014013 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.00996767240576446, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011455767671577632, "completions/clipped_ratio": 0.0625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 14.0, "completions/mean_terminated_length": 13.866667747497559, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.6236467249691486, "epoch": 0.00078, "frac_reward_zero_std": 0.0, "grad_norm": 0.7034565210342407, "kl": 0.20543900597840548, "learning_rate": 9.99977740929988e-06, "loss": -0.0375, "num_tokens": 1741045.0, "reward": -0.0031250007450580597, "reward_std": 0.30141550302505493, "rewards/rollout_reward_func/mean": -0.0031250007450580597, "rewards/rollout_reward_func/std": 0.29125356674194336, "sampling/importance_sampling_ratio/max": 1.6532666683197021, "sampling/importance_sampling_ratio/mean": 0.9591487050056458, "sampling/importance_sampling_ratio/min": 2.4133162152602e-41, "sampling/sampling_logp_difference/max": 33.43802261352539, "sampling/sampling_logp_difference/mean": 0.6354968547821045, "step": 39, "step_time": 18.63147399299487 }, { "clip_ratio/high_max": 0.001953125, "clip_ratio/high_mean": 0.0009765625, "clip_ratio/low_mean": 0.0034722222480922937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004448784748092294, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 16.1875, "completions/mean_terminated_length": 14.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.6384543851017952, "epoch": 0.0008, "frac_reward_zero_std": 0.0, "grad_norm": 0.5525887608528137, "kl": 0.46802808821666986, "learning_rate": 9.999604286244655e-06, "loss": -0.0248, "num_tokens": 1785818.0, "reward": -0.13750001788139343, "reward_std": 0.273576557636261, "rewards/rollout_reward_func/mean": -0.13750001788139343, "rewards/rollout_reward_func/std": 0.2720887064933777, "sampling/importance_sampling_ratio/max": 1.6926090717315674, "sampling/importance_sampling_ratio/mean": 0.9045075178146362, "sampling/importance_sampling_ratio/min": 7.541191011765291e-26, "sampling/sampling_logp_difference/max": 19.793434143066406, "sampling/sampling_logp_difference/mean": 0.48404866456985474, "step": 40, "step_time": 17.632873179987655 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0014204545877873898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026704545598477125, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.40625, "completions/mean_terminated_length": 16.760000228881836, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.8357867393642664, "epoch": 0.00082, "frac_reward_zero_std": 0.0, "grad_norm": 0.5087578296661377, "kl": 0.07389803987462074, "learning_rate": 9.999381703374151e-06, "loss": -0.1011, "num_tokens": 1830347.0, "reward": -0.09375, "reward_std": 0.27669018507003784, "rewards/rollout_reward_func/mean": -0.09375, "rewards/rollout_reward_func/std": 0.29395797848701477, "sampling/importance_sampling_ratio/max": 1.7966524362564087, "sampling/importance_sampling_ratio/mean": 0.7763824462890625, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.00799560546875, "sampling/sampling_logp_difference/mean": 1.1789870262145996, "step": 41, "step_time": 18.479260904045077 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 14.96875, "completions/mean_terminated_length": 13.17391300201416, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.6373399700969458, "epoch": 0.00084, "frac_reward_zero_std": 0.0, "grad_norm": 0.5095486640930176, "kl": 0.07425376819446683, "learning_rate": 9.999109663624387e-06, "loss": -0.0679, "num_tokens": 1872090.0, "reward": 0.03125, "reward_std": 0.34633269906044006, "rewards/rollout_reward_func/mean": 0.03125, "rewards/rollout_reward_func/std": 0.3325730562210083, "sampling/importance_sampling_ratio/max": 1.6861286163330078, "sampling/importance_sampling_ratio/mean": 0.9842855334281921, "sampling/importance_sampling_ratio/min": 8.298489505731567e-42, "sampling/sampling_logp_difference/max": 33.570106506347656, "sampling/sampling_logp_difference/mean": 0.5733375549316406, "step": 42, "step_time": 18.72764529196138 }, { "clip_ratio/high_max": 0.0056933199521154165, "clip_ratio/high_mean": 0.0028466599760577083, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028466599760577083, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 16.75, "completions/mean_terminated_length": 15.703703880310059, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.6878395844250917, "epoch": 0.00086, "frac_reward_zero_std": 0.0, "grad_norm": 0.5654966831207275, "kl": 0.1392933214083314, "learning_rate": 9.99878817058375e-06, "loss": -0.049, "num_tokens": 1917357.0, "reward": -0.10625000298023224, "reward_std": 0.221847265958786, "rewards/rollout_reward_func/mean": -0.10625000298023224, "rewards/rollout_reward_func/std": 0.2381887286901474, "sampling/importance_sampling_ratio/max": 1.7386271953582764, "sampling/importance_sampling_ratio/mean": 0.8362831473350525, "sampling/importance_sampling_ratio/min": 3.1991643940535574e-42, "sampling/sampling_logp_difference/max": 33.18673324584961, "sampling/sampling_logp_difference/mean": 0.9068950414657593, "step": 43, "step_time": 18.103344882969395 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032031249720603228, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 15.96875, "completions/mean_terminated_length": 15.260869979858398, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7210715040564537, "epoch": 0.00088, "frac_reward_zero_std": 0.0, "grad_norm": 0.6250632405281067, "kl": 0.1402771376306191, "learning_rate": 9.998417228492952e-06, "loss": 0.0647, "num_tokens": 1963095.0, "reward": -0.15625, "reward_std": 0.2553158104419708, "rewards/rollout_reward_func/mean": -0.15625, "rewards/rollout_reward_func/std": 0.2626631557941437, "sampling/importance_sampling_ratio/max": 2.0873937606811523, "sampling/importance_sampling_ratio/mean": 0.8514653444290161, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.30154037475586, "sampling/sampling_logp_difference/mean": 0.9060221910476685, "step": 44, "step_time": 18.77604677400086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0012019231216982007, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012019231216982007, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 16.5, "completions/mean_terminated_length": 13.458333969116211, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7678162064403296, "epoch": 0.0009, "frac_reward_zero_std": 0.0, "grad_norm": 0.763396143913269, "kl": 0.09200582304038107, "learning_rate": 9.99799684224497e-06, "loss": -0.0355, "num_tokens": 2007803.0, "reward": -0.03437500447034836, "reward_std": 0.2670585513114929, "rewards/rollout_reward_func/mean": -0.03437500447034836, "rewards/rollout_reward_func/std": 0.2846722900867462, "sampling/importance_sampling_ratio/max": 1.5547850131988525, "sampling/importance_sampling_ratio/mean": 0.8561392426490784, "sampling/importance_sampling_ratio/min": 5.997557427310217e-43, "sampling/sampling_logp_difference/max": 32.60765075683594, "sampling/sampling_logp_difference/mean": 1.074091911315918, "step": 45, "step_time": 17.866422290957416 }, { "clip_ratio/high_max": 0.00617647054605186, "clip_ratio/high_mean": 0.00308823527302593, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00308823527302593, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 16.28125, "completions/mean_terminated_length": 15.125, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7729356717318296, "epoch": 0.00092, "frac_reward_zero_std": 0.0, "grad_norm": 0.5737928748130798, "kl": 0.4691641665995121, "learning_rate": 9.997527017384983e-06, "loss": -0.033, "num_tokens": 2052244.0, "reward": -0.09062500298023224, "reward_std": 0.29058098793029785, "rewards/rollout_reward_func/mean": -0.09062500298023224, "rewards/rollout_reward_func/std": 0.3155736029148102, "sampling/importance_sampling_ratio/max": 1.3794009685516357, "sampling/importance_sampling_ratio/mean": 0.6315082311630249, "sampling/importance_sampling_ratio/min": 9.30074160766907e-39, "sampling/sampling_logp_difference/max": 33.12974166870117, "sampling/sampling_logp_difference/mean": 1.119800329208374, "step": 46, "step_time": 18.646205128985457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 15.625, "completions/mean_terminated_length": 13.272727966308594, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.6073273606598377, "epoch": 0.00094, "frac_reward_zero_std": 0.0, "grad_norm": 0.6136278510093689, "kl": 0.09560762147884816, "learning_rate": 9.997007760110298e-06, "loss": 0.0353, "num_tokens": 2095682.0, "reward": -0.06562499701976776, "reward_std": 0.25442248582839966, "rewards/rollout_reward_func/mean": -0.06562499701976776, "rewards/rollout_reward_func/std": 0.27192193269729614, "sampling/importance_sampling_ratio/max": 1.5851274728775024, "sampling/importance_sampling_ratio/mean": 0.8684334754943848, "sampling/importance_sampling_ratio/min": 1.7868985548580794e-17, "sampling/sampling_logp_difference/max": 18.141855239868164, "sampling/sampling_logp_difference/mean": 0.321379691362381, "step": 47, "step_time": 17.430216189983184 }, { "clip_ratio/high_max": 0.006770833628252149, "clip_ratio/high_mean": 0.0033854168141260743, "clip_ratio/low_mean": 0.018177083344198763, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.021562500158324838, "completions/clipped_ratio": 0.15625, "completions/max_length": 28.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 15.3125, "completions/mean_terminated_length": 15.185185432434082, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7860593795776367, "epoch": 0.00096, "frac_reward_zero_std": 0.0, "grad_norm": 0.7489107847213745, "kl": 0.553908551344648, "learning_rate": 9.996439077270269e-06, "loss": -0.0652, "num_tokens": 2140154.0, "reward": -0.11562500894069672, "reward_std": 0.24994677305221558, "rewards/rollout_reward_func/mean": -0.11562500894069672, "rewards/rollout_reward_func/std": 0.250302255153656, "sampling/importance_sampling_ratio/max": 1.808884620666504, "sampling/importance_sampling_ratio/mean": 0.8113023042678833, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.46635055541992, "sampling/sampling_logp_difference/mean": 1.2337117195129395, "step": 48, "step_time": 17.78923621099966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 16.65625, "completions/mean_terminated_length": 14.25, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.6964225452393293, "epoch": 0.00098, "frac_reward_zero_std": 0.0, "grad_norm": 0.42079946398735046, "kl": 0.20602299692109227, "learning_rate": 9.995820976366208e-06, "loss": 0.2003, "num_tokens": 2185996.0, "reward": -0.08437500149011612, "reward_std": 0.24799031019210815, "rewards/rollout_reward_func/mean": -0.08437500149011612, "rewards/rollout_reward_func/std": 0.2713281214237213, "sampling/importance_sampling_ratio/max": 1.8333933353424072, "sampling/importance_sampling_ratio/mean": 0.8454562425613403, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 32.671104431152344, "sampling/sampling_logp_difference/mean": 0.731881856918335, "step": 49, "step_time": 19.6879031080025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 19.0, "completions/mean_terminated_length": 16.0, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.8804360199719667, "epoch": 0.001, "frac_reward_zero_std": 0.0, "grad_norm": 0.36105501651763916, "kl": 0.07642816641600803, "learning_rate": 9.995153465551283e-06, "loss": -0.0246, "num_tokens": 2231024.0, "reward": -0.20000001788139343, "reward_std": 0.2438179850578308, "rewards/rollout_reward_func/mean": -0.20000001788139343, "rewards/rollout_reward_func/std": 0.2565276622772217, "sampling/importance_sampling_ratio/max": 1.6398296356201172, "sampling/importance_sampling_ratio/mean": 0.6633528470993042, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.00759506225586, "sampling/sampling_logp_difference/mean": 1.1237614154815674, "step": 50, "step_time": 18.229174644016894 }, { "clip_ratio/high_max": 0.006666666828095913, "clip_ratio/high_mean": 0.004410919500514865, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004410919500514865, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 15.59375, "completions/mean_terminated_length": 13.769230842590332, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8090154156088829, "epoch": 0.00102, "frac_reward_zero_std": 0.0, "grad_norm": 0.547369122505188, "kl": 0.6521737047005445, "learning_rate": 9.994436553630411e-06, "loss": -0.0839, "num_tokens": 2275261.0, "reward": -0.12187500298023224, "reward_std": 0.23174212872982025, "rewards/rollout_reward_func/mean": -0.12187500298023224, "rewards/rollout_reward_func/std": 0.27207016944885254, "sampling/importance_sampling_ratio/max": 1.9818400144577026, "sampling/importance_sampling_ratio/mean": 0.6950240135192871, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.648799896240234, "sampling/sampling_logp_difference/mean": 1.2598581314086914, "step": 51, "step_time": 18.73514803101716 }, { "clip_ratio/high_max": 0.006469253916293383, "clip_ratio/high_mean": 0.0032346269581466913, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032346269581466913, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 14.84375, "completions/mean_terminated_length": 14.17241382598877, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.6761191785335541, "epoch": 0.00104, "frac_reward_zero_std": 0.0, "grad_norm": 0.392436146736145, "kl": 0.42358784982934594, "learning_rate": 9.993670250060152e-06, "loss": -0.1196, "num_tokens": 2319563.0, "reward": -0.109375, "reward_std": 0.27844253182411194, "rewards/rollout_reward_func/mean": -0.109375, "rewards/rollout_reward_func/std": 0.2763347327709198, "sampling/importance_sampling_ratio/max": 1.843059778213501, "sampling/importance_sampling_ratio/mean": 0.7400991916656494, "sampling/importance_sampling_ratio/min": 1.814681511300638e-42, "sampling/sampling_logp_difference/max": 31.88886070251465, "sampling/sampling_logp_difference/mean": 0.9520541429519653, "step": 52, "step_time": 17.625572700009798 }, { "clip_ratio/high_max": 0.008795229718089104, "clip_ratio/high_mean": 0.004397614859044552, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004397614859044552, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.03125, "completions/mean_terminated_length": 17.380952835083008, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9568933397531509, "epoch": 0.00106, "frac_reward_zero_std": 0.0, "grad_norm": 0.2904786467552185, "kl": 0.32567093055695295, "learning_rate": 9.99285456494856e-06, "loss": 0.0187, "num_tokens": 2365255.0, "reward": -0.13125000894069672, "reward_std": 0.3177714943885803, "rewards/rollout_reward_func/mean": -0.13125000894069672, "rewards/rollout_reward_func/std": 0.3187045454978943, "sampling/importance_sampling_ratio/max": 2.0234522819519043, "sampling/importance_sampling_ratio/mean": 0.5103186368942261, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.9514045715332, "sampling/sampling_logp_difference/mean": 1.4995191097259521, "step": 53, "step_time": 19.309910511015914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0028466599760577083, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028466599760577083, "completions/clipped_ratio": 0.09375, "completions/max_length": 28.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 13.5625, "completions/mean_terminated_length": 13.310344696044922, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.6009986083954573, "epoch": 0.00108, "frac_reward_zero_std": 0.0, "grad_norm": 0.6462810635566711, "kl": 2.9028957701520994, "learning_rate": 9.991989509055075e-06, "loss": -0.1205, "num_tokens": 2407610.0, "reward": -0.07500000298023224, "reward_std": 0.31726014614105225, "rewards/rollout_reward_func/mean": -0.07500000298023224, "rewards/rollout_reward_func/std": 0.31520602107048035, "sampling/importance_sampling_ratio/max": 2.038015365600586, "sampling/importance_sampling_ratio/mean": 0.7069606781005859, "sampling/importance_sampling_ratio/min": 3.2492407475591605e-37, "sampling/sampling_logp_difference/max": 32.03176498413086, "sampling/sampling_logp_difference/mean": 0.8884064555168152, "step": 54, "step_time": 17.991187758932938 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011574074160307646, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 16.8125, "completions/mean_terminated_length": 16.34482765197754, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7793903946876526, "epoch": 0.0011, "frac_reward_zero_std": 0.0, "grad_norm": 0.4308735132217407, "kl": 0.34391536843031645, "learning_rate": 9.991075093790372e-06, "loss": -0.0324, "num_tokens": 2452907.0, "reward": -0.140625, "reward_std": 0.27962490916252136, "rewards/rollout_reward_func/mean": -0.140625, "rewards/rollout_reward_func/std": 0.288331538438797, "sampling/importance_sampling_ratio/max": 2.0240211486816406, "sampling/importance_sampling_ratio/mean": 0.7660483121871948, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.101356506347656, "sampling/sampling_logp_difference/mean": 1.1996617317199707, "step": 55, "step_time": 17.794971701980103 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010775862028822303, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 16.21875, "completions/mean_terminated_length": 13.719999313354492, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7795454803854227, "epoch": 0.00112, "frac_reward_zero_std": 0.0, "grad_norm": 0.5098366737365723, "kl": 0.17088248790241778, "learning_rate": 9.990111331216204e-06, "loss": 0.0759, "num_tokens": 2497818.0, "reward": -0.02499999850988388, "reward_std": 0.2755615711212158, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.285114586353302, "sampling/importance_sampling_ratio/max": 1.7385164499282837, "sampling/importance_sampling_ratio/mean": 0.6270055770874023, "sampling/importance_sampling_ratio/min": 2.2392749459910577e-42, "sampling/sampling_logp_difference/max": 32.769493103027344, "sampling/sampling_logp_difference/mean": 1.1691169738769531, "step": 56, "step_time": 18.53233401599573 }, { "clip_ratio/high_max": 0.004464285913854837, "clip_ratio/high_mean": 0.0022321429569274187, "clip_ratio/low_mean": 0.003309729159809649, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005541872116737068, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.90625, "completions/mean_terminated_length": 16.703702926635742, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9919640775769949, "epoch": 0.00114, "frac_reward_zero_std": 0.0, "grad_norm": 0.3060133755207062, "kl": 0.14134800049941987, "learning_rate": 9.989098234045254e-06, "loss": -0.1237, "num_tokens": 2543242.0, "reward": -0.08124999701976776, "reward_std": 0.3350231647491455, "rewards/rollout_reward_func/mean": -0.08124999701976776, "rewards/rollout_reward_func/std": 0.329650342464447, "sampling/importance_sampling_ratio/max": 2.3174967765808105, "sampling/importance_sampling_ratio/mean": 0.5282878279685974, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.21797561645508, "sampling/sampling_logp_difference/mean": 1.8543156385421753, "step": 57, "step_time": 19.086933182014036 }, { "clip_ratio/high_max": 0.0031250000465661287, "clip_ratio/high_mean": 0.0015625000232830644, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009375000023283064, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 16.875, "completions/mean_terminated_length": 15.260869979858398, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8617880791425705, "epoch": 0.00116, "frac_reward_zero_std": 0.0, "grad_norm": 0.5175003409385681, "kl": 0.3897188750561327, "learning_rate": 9.988035815640953e-06, "loss": -0.0275, "num_tokens": 2588337.0, "reward": -0.109375, "reward_std": 0.29230326414108276, "rewards/rollout_reward_func/mean": -0.109375, "rewards/rollout_reward_func/std": 0.2944205701351166, "sampling/importance_sampling_ratio/max": 1.8001184463500977, "sampling/importance_sampling_ratio/mean": 0.6417454481124878, "sampling/importance_sampling_ratio/min": 5.1217458871072064e-42, "sampling/sampling_logp_difference/max": 32.45234680175781, "sampling/sampling_logp_difference/mean": 1.0458823442459106, "step": 58, "step_time": 18.554628895013593 }, { "clip_ratio/high_max": 0.00808566459454596, "clip_ratio/high_mean": 0.00404283229727298, "clip_ratio/low_mean": 0.004964465042576194, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009007297339849174, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 17.3125, "completions/mean_terminated_length": 15.75999927520752, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9553849324584007, "epoch": 0.00118, "frac_reward_zero_std": 0.0, "grad_norm": 0.33320605754852295, "kl": 0.12742998590692878, "learning_rate": 9.986924090017322e-06, "loss": 0.0298, "num_tokens": 2635054.0, "reward": -0.16250000894069672, "reward_std": 0.28963130712509155, "rewards/rollout_reward_func/mean": -0.16250000894069672, "rewards/rollout_reward_func/std": 0.2859618663787842, "sampling/importance_sampling_ratio/max": 1.6478148698806763, "sampling/importance_sampling_ratio/mean": 0.4600265324115753, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.546104431152344, "sampling/sampling_logp_difference/mean": 1.4871835708618164, "step": 59, "step_time": 19.19067249301588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004096790915355086, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004096790915355086, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 16.03125, "completions/mean_terminated_length": 15.423077583312988, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7490036971867085, "epoch": 0.0012, "frac_reward_zero_std": 0.0, "grad_norm": 0.516810953617096, "kl": 0.10992411989718676, "learning_rate": 9.985763071838774e-06, "loss": -0.0555, "num_tokens": 2679338.0, "reward": -0.06562499701976776, "reward_std": 0.3062734305858612, "rewards/rollout_reward_func/mean": -0.06562499701976776, "rewards/rollout_reward_func/std": 0.2979601323604584, "sampling/importance_sampling_ratio/max": 2.0056958198547363, "sampling/importance_sampling_ratio/mean": 0.72804194688797, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 32.31302261352539, "sampling/sampling_logp_difference/mean": 1.2942402362823486, "step": 60, "step_time": 18.161147000049823 }, { "clip_ratio/high_max": 0.010775691829621792, "clip_ratio/high_mean": 0.005387845914810896, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005387845914810896, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 25.0, "completions/mean_length": 15.0, "completions/mean_terminated_length": 13.538461685180664, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.9135260991752148, "epoch": 0.00122, "frac_reward_zero_std": 0.0, "grad_norm": 0.4998561441898346, "kl": 0.17559171025641263, "learning_rate": 9.98455277641992e-06, "loss": 0.0183, "num_tokens": 2724168.0, "reward": 0.0, "reward_std": 0.28164365887641907, "rewards/rollout_reward_func/mean": 0.0, "rewards/rollout_reward_func/std": 0.2839809060096741, "sampling/importance_sampling_ratio/max": 1.3575124740600586, "sampling/importance_sampling_ratio/mean": 0.5742080807685852, "sampling/importance_sampling_ratio/min": 9.542842542052004e-43, "sampling/sampling_logp_difference/max": 33.43802261352539, "sampling/sampling_logp_difference/mean": 1.7244447469711304, "step": 61, "step_time": 18.773632473035832 }, { "clip_ratio/high_max": 0.0047186610754579306, "clip_ratio/high_mean": 0.0023593305377289653, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023593305377289653, "completions/clipped_ratio": 0.40625, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.40625, "completions/mean_terminated_length": 14.421052932739258, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8566051255911589, "epoch": 0.00124, "frac_reward_zero_std": 0.0, "grad_norm": 0.46803030371665955, "kl": 0.12423868011683226, "learning_rate": 9.983293219725379e-06, "loss": -0.0929, "num_tokens": 2768797.0, "reward": -0.04374999925494194, "reward_std": 0.31975090503692627, "rewards/rollout_reward_func/mean": -0.04374999925494194, "rewards/rollout_reward_func/std": 0.3181980550289154, "sampling/importance_sampling_ratio/max": 2.1665337085723877, "sampling/importance_sampling_ratio/mean": 0.6284246444702148, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.9300537109375, "sampling/sampling_logp_difference/mean": 1.2768681049346924, "step": 62, "step_time": 19.041066321980907 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011574074160307646, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.1875, "completions/mean_terminated_length": 18.31999969482422, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0184374749660492, "epoch": 0.00126, "frac_reward_zero_std": 0.0, "grad_norm": 0.327326238155365, "kl": 0.08377526979893446, "learning_rate": 9.981984418369552e-06, "loss": -0.079, "num_tokens": 2814454.0, "reward": -0.15312501788139343, "reward_std": 0.27512767910957336, "rewards/rollout_reward_func/mean": -0.15312501788139343, "rewards/rollout_reward_func/std": 0.29728272557258606, "sampling/importance_sampling_ratio/max": 1.9981598854064941, "sampling/importance_sampling_ratio/mean": 0.5449277758598328, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.780479431152344, "sampling/sampling_logp_difference/mean": 1.5302561521530151, "step": 63, "step_time": 19.71820322998974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 16.125, "completions/mean_terminated_length": 14.000000953674316, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8732791505753994, "epoch": 0.00128, "frac_reward_zero_std": 0.0, "grad_norm": 0.6467760801315308, "kl": 0.2797319176606834, "learning_rate": 9.980626389616414e-06, "loss": -0.0567, "num_tokens": 2858924.0, "reward": 0.01875000074505806, "reward_std": 0.25117406249046326, "rewards/rollout_reward_func/mean": 0.01875000074505806, "rewards/rollout_reward_func/std": 0.24943484365940094, "sampling/importance_sampling_ratio/max": 2.1540279388427734, "sampling/importance_sampling_ratio/mean": 0.5977157950401306, "sampling/importance_sampling_ratio/min": 2.0739217272007293e-43, "sampling/sampling_logp_difference/max": 31.83727264404297, "sampling/sampling_logp_difference/mean": 1.4227550029754639, "step": 64, "step_time": 18.11158631001308 }, { "clip_ratio/high_max": 0.0116257444024086, "clip_ratio/high_mean": 0.0058128722012043, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007062872173264623, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 17.125, "completions/mean_terminated_length": 15.115385055541992, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9854319542646408, "epoch": 0.0013, "frac_reward_zero_std": 0.0, "grad_norm": 0.6895468235015869, "kl": 0.24782478879205883, "learning_rate": 9.979219151379285e-06, "loss": -0.201, "num_tokens": 2905044.0, "reward": -0.125, "reward_std": 0.2995797395706177, "rewards/rollout_reward_func/mean": -0.125, "rewards/rollout_reward_func/std": 0.29946190118789673, "sampling/importance_sampling_ratio/max": 2.550102472305298, "sampling/importance_sampling_ratio/mean": 0.6183609366416931, "sampling/importance_sampling_ratio/min": 3.44719422223905e-43, "sampling/sampling_logp_difference/max": 32.86754608154297, "sampling/sampling_logp_difference/mean": 1.7825665473937988, "step": 65, "step_time": 18.432699543001945 }, { "clip_ratio/high_max": 0.0077173912432044744, "clip_ratio/high_mean": 0.0038586956216022372, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038586956216022372, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 18.15625, "completions/mean_terminated_length": 15.833333969116211, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0490214619785547, "epoch": 0.00132, "frac_reward_zero_std": 0.0, "grad_norm": 0.35932064056396484, "kl": 0.15446288033854216, "learning_rate": 9.977762722220585e-06, "loss": -0.0398, "num_tokens": 2950397.0, "reward": -0.02500000037252903, "reward_std": 0.325346440076828, "rewards/rollout_reward_func/mean": -0.02500000037252903, "rewards/rollout_reward_func/std": 0.3172462284564972, "sampling/importance_sampling_ratio/max": 2.228111505508423, "sampling/importance_sampling_ratio/mean": 0.5810765027999878, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.2738037109375, "sampling/sampling_logp_difference/mean": 1.8664799928665161, "step": 66, "step_time": 18.915184055062127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.009548611124046147, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009548611124046147, "completions/clipped_ratio": 0.125, "completions/max_length": 30.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 15.75, "completions/mean_terminated_length": 15.714286804199219, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7380697466433048, "epoch": 0.00134, "frac_reward_zero_std": 0.0, "grad_norm": 0.3962846100330353, "kl": 0.13459058827720582, "learning_rate": 9.976257121351602e-06, "loss": -0.1552, "num_tokens": 2994041.0, "reward": 0.07500000298023224, "reward_std": 0.3174649477005005, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.31004682183265686, "sampling/importance_sampling_ratio/max": 2.4542455673217773, "sampling/importance_sampling_ratio/mean": 0.7130502462387085, "sampling/importance_sampling_ratio/min": 6.1432340372655675e-34, "sampling/sampling_logp_difference/max": 31.449661254882812, "sampling/sampling_logp_difference/mean": 1.098301649093628, "step": 67, "step_time": 17.693776582964347 }, { "clip_ratio/high_max": 0.010937500512227416, "clip_ratio/high_mean": 0.005468750256113708, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005468750256113708, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.15625, "completions/mean_terminated_length": 14.30434799194336, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.8328164182603359, "epoch": 0.00136, "frac_reward_zero_std": 0.0, "grad_norm": 0.28603625297546387, "kl": 0.1432439957279712, "learning_rate": 9.974702368632228e-06, "loss": -0.1914, "num_tokens": 3038023.0, "reward": -0.05312499403953552, "reward_std": 0.3376256823539734, "rewards/rollout_reward_func/mean": -0.05312499403953552, "rewards/rollout_reward_func/std": 0.3473255932331085, "sampling/importance_sampling_ratio/max": 1.8803492784500122, "sampling/importance_sampling_ratio/mean": 0.6438254117965698, "sampling/importance_sampling_ratio/min": 1.0899299455518427e-41, "sampling/sampling_logp_difference/max": 32.11851119995117, "sampling/sampling_logp_difference/mean": 1.2446675300598145, "step": 68, "step_time": 18.55532671500987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.09375, "completions/mean_terminated_length": 14.833333969116211, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.0515582002699375, "epoch": 0.00138, "frac_reward_zero_std": 0.0, "grad_norm": 0.6029288172721863, "kl": 0.12366867205128074, "learning_rate": 9.973098484570702e-06, "loss": -0.113, "num_tokens": 3083279.0, "reward": -0.02500000037252903, "reward_std": 0.3251707851886749, "rewards/rollout_reward_func/mean": -0.02500000037252903, "rewards/rollout_reward_func/std": 0.3121207356452942, "sampling/importance_sampling_ratio/max": 2.2251155376434326, "sampling/importance_sampling_ratio/mean": 0.6629937887191772, "sampling/importance_sampling_ratio/min": 4.203895392974451e-45, "sampling/sampling_logp_difference/max": 33.19966506958008, "sampling/sampling_logp_difference/mean": 1.7184499502182007, "step": 69, "step_time": 18.449921495019225 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033118206774815917, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 17.9375, "completions/mean_terminated_length": 16.760000228881836, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.939851077273488, "epoch": 0.0014, "frac_reward_zero_std": 0.0, "grad_norm": 0.6057099103927612, "kl": 0.17996075190603733, "learning_rate": 9.97144549032334e-06, "loss": -0.1383, "num_tokens": 3128805.0, "reward": -0.1093750074505806, "reward_std": 0.31600072979927063, "rewards/rollout_reward_func/mean": -0.1093750074505806, "rewards/rollout_reward_func/std": 0.3165941536426544, "sampling/importance_sampling_ratio/max": 2.8154995441436768, "sampling/importance_sampling_ratio/mean": 0.5471562147140503, "sampling/importance_sampling_ratio/min": 3.461207206882298e-43, "sampling/sampling_logp_difference/max": 32.97354507446289, "sampling/sampling_logp_difference/mean": 1.665574312210083, "step": 70, "step_time": 18.76303921501676 }, { "clip_ratio/high_max": 0.013953202171251178, "clip_ratio/high_mean": 0.006976601085625589, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006976601085625589, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.40625, "completions/mean_terminated_length": 16.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9043285883963108, "epoch": 0.00142, "frac_reward_zero_std": 0.0, "grad_norm": 0.30015766620635986, "kl": 0.1592502638231963, "learning_rate": 9.969743407694254e-06, "loss": -0.1867, "num_tokens": 3172971.0, "reward": 0.05000000447034836, "reward_std": 0.32524919509887695, "rewards/rollout_reward_func/mean": 0.05000000447034836, "rewards/rollout_reward_func/std": 0.32229021191596985, "sampling/importance_sampling_ratio/max": 1.8314038515090942, "sampling/importance_sampling_ratio/mean": 0.4726044535636902, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.86754608154297, "sampling/sampling_logp_difference/mean": 1.5190694332122803, "step": 71, "step_time": 18.86370393598918 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 17.53125, "completions/mean_terminated_length": 15.375, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8811447434127331, "epoch": 0.00144, "frac_reward_zero_std": 0.0, "grad_norm": 0.5429362058639526, "kl": 0.7765106449369341, "learning_rate": 9.967992259135063e-06, "loss": -0.1324, "num_tokens": 3218524.0, "reward": -0.08125000447034836, "reward_std": 0.3258792757987976, "rewards/rollout_reward_func/mean": -0.08125000447034836, "rewards/rollout_reward_func/std": 0.3227277994155884, "sampling/importance_sampling_ratio/max": 1.82301664352417, "sampling/importance_sampling_ratio/mean": 0.5391995310783386, "sampling/importance_sampling_ratio/min": 8.433014158306749e-42, "sampling/sampling_logp_difference/max": 32.28604507446289, "sampling/sampling_logp_difference/mean": 1.355555772781372, "step": 72, "step_time": 18.337855826001032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0039242461789399385, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0039242461789399385, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 16.125, "completions/mean_terminated_length": 14.59999942779541, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8054924719035625, "epoch": 0.00146, "frac_reward_zero_std": 0.0, "grad_norm": 0.5236900448799133, "kl": 0.2835140251554549, "learning_rate": 9.9661920677446e-06, "loss": -0.1486, "num_tokens": 3263020.0, "reward": -0.07500000298023224, "reward_std": 0.31941115856170654, "rewards/rollout_reward_func/mean": -0.07500000298023224, "rewards/rollout_reward_func/std": 0.3242858052253723, "sampling/importance_sampling_ratio/max": 1.8416184186935425, "sampling/importance_sampling_ratio/mean": 0.6946564316749573, "sampling/importance_sampling_ratio/min": 7.162040655059533e-39, "sampling/sampling_logp_difference/max": 32.23359680175781, "sampling/sampling_logp_difference/mean": 1.0821279287338257, "step": 73, "step_time": 20.004156581984716 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0025297619868069887, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037797619588673115, "completions/clipped_ratio": 0.3125, "completions/max_length": 48.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 21.46875, "completions/mean_terminated_length": 17.68181800842285, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9310360364615917, "epoch": 0.00148, "frac_reward_zero_std": 0.0, "grad_norm": 0.20439179241657257, "kl": 0.1508938237093389, "learning_rate": 9.964342857268609e-06, "loss": -0.0984, "num_tokens": 3308751.0, "reward": -0.16562500596046448, "reward_std": 0.4134608507156372, "rewards/rollout_reward_func/mean": -0.16562500596046448, "rewards/rollout_reward_func/std": 0.40529507398605347, "sampling/importance_sampling_ratio/max": 1.6546244621276855, "sampling/importance_sampling_ratio/mean": 0.2889052629470825, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.85135269165039, "sampling/sampling_logp_difference/mean": 1.6392474174499512, "step": 74, "step_time": 19.668856952965143 }, { "clip_ratio/high_max": 0.015395220601931214, "clip_ratio/high_mean": 0.007697610300965607, "clip_ratio/low_mean": 0.004640379571355879, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012337989872321486, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.65625, "completions/mean_terminated_length": 16.782609939575195, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9728524275124073, "epoch": 0.0015, "frac_reward_zero_std": 0.0, "grad_norm": 0.2422945201396942, "kl": 0.18815409997478127, "learning_rate": 9.962444652099422e-06, "loss": -0.1841, "num_tokens": 3354648.0, "reward": -0.12187500298023224, "reward_std": 0.3403671681880951, "rewards/rollout_reward_func/mean": -0.12187500298023224, "rewards/rollout_reward_func/std": 0.33286088705062866, "sampling/importance_sampling_ratio/max": 2.0247178077697754, "sampling/importance_sampling_ratio/mean": 0.4604012370109558, "sampling/importance_sampling_ratio/min": 6.978466352337589e-42, "sampling/sampling_logp_difference/max": 32.75444030761719, "sampling/sampling_logp_difference/mean": 1.6138484477996826, "step": 75, "step_time": 20.461681162007153 }, { "clip_ratio/high_max": 0.009591224836185575, "clip_ratio/high_mean": 0.004795612418092787, "clip_ratio/low_mean": 0.0024999999441206455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007295612362213433, "completions/clipped_ratio": 0.25, "completions/max_length": 35.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 18.21875, "completions/mean_terminated_length": 16.041667938232422, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.019128443673253, "epoch": 0.00152, "frac_reward_zero_std": 0.0, "grad_norm": 0.3498808443546295, "kl": 0.1951793225016445, "learning_rate": 9.960497477275653e-06, "loss": -0.105, "num_tokens": 3400174.0, "reward": -0.07500000298023224, "reward_std": 0.35815954208374023, "rewards/rollout_reward_func/mean": -0.07500000298023224, "rewards/rollout_reward_func/std": 0.3528684377670288, "sampling/importance_sampling_ratio/max": 1.732198715209961, "sampling/importance_sampling_ratio/mean": 0.5566525459289551, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.75760269165039, "sampling/sampling_logp_difference/mean": 1.966657042503357, "step": 76, "step_time": 18.955548856014502 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0026607790496200323, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026607790496200323, "completions/clipped_ratio": 0.09375, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 17.59375, "completions/mean_terminated_length": 17.20689582824707, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8354981057345867, "epoch": 0.00154, "frac_reward_zero_std": 0.0, "grad_norm": 0.6819092035293579, "kl": 1.6971925315447152, "learning_rate": 9.95850135848185e-06, "loss": -0.0457, "num_tokens": 3445238.0, "reward": -0.08750000596046448, "reward_std": 0.27958542108535767, "rewards/rollout_reward_func/mean": -0.08750000596046448, "rewards/rollout_reward_func/std": 0.30134108662605286, "sampling/importance_sampling_ratio/max": 2.0660018920898438, "sampling/importance_sampling_ratio/mean": 0.5125373005867004, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.036006927490234, "sampling/sampling_logp_difference/mean": 1.4916764497756958, "step": 77, "step_time": 18.650451557987253 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.0012019231216982007, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026223777094855905, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 19.53125, "completions/mean_terminated_length": 16.846155166625977, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0191352013498545, "epoch": 0.00156, "frac_reward_zero_std": 0.0, "grad_norm": 0.2921481728553772, "kl": 0.31131338118575513, "learning_rate": 9.956456322048174e-06, "loss": -0.0838, "num_tokens": 3490794.0, "reward": -0.11562500894069672, "reward_std": 0.3236559331417084, "rewards/rollout_reward_func/mean": -0.11562500894069672, "rewards/rollout_reward_func/std": 0.35113582015037537, "sampling/importance_sampling_ratio/max": 2.2429563999176025, "sampling/importance_sampling_ratio/mean": 0.38677161931991577, "sampling/importance_sampling_ratio/min": 4.624284932271896e-44, "sampling/sampling_logp_difference/max": 33.17109680175781, "sampling/sampling_logp_difference/mean": 1.9739351272583008, "step": 78, "step_time": 21.8375731529959 }, { "clip_ratio/high_max": 0.004759339150041342, "clip_ratio/high_mean": 0.002379669575020671, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002379669575020671, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 17.28125, "completions/mean_terminated_length": 15.592592239379883, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9944030120968819, "epoch": 0.00158, "frac_reward_zero_std": 0.0, "grad_norm": 0.274970144033432, "kl": 0.31029514758847654, "learning_rate": 9.954362394950035e-06, "loss": -0.005, "num_tokens": 3536693.0, "reward": -0.03437499701976776, "reward_std": 0.27632761001586914, "rewards/rollout_reward_func/mean": -0.03437499701976776, "rewards/rollout_reward_func/std": 0.27192190289497375, "sampling/importance_sampling_ratio/max": 2.014986038208008, "sampling/importance_sampling_ratio/mean": 0.4954504668712616, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.27323913574219, "sampling/sampling_logp_difference/mean": 1.9817463159561157, "step": 79, "step_time": 18.751091911981348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.0, "completions/mean_terminated_length": 15.074073791503906, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7156291529536247, "epoch": 0.0016, "frac_reward_zero_std": 0.0, "grad_norm": 0.4943658709526062, "kl": 0.28480181423947215, "learning_rate": 9.952219604807746e-06, "loss": -0.1663, "num_tokens": 3580901.0, "reward": -0.01562499813735485, "reward_std": 0.31406766176223755, "rewards/rollout_reward_func/mean": -0.01562499813735485, "rewards/rollout_reward_func/std": 0.308073490858078, "sampling/importance_sampling_ratio/max": 2.676137924194336, "sampling/importance_sampling_ratio/mean": 0.7172547578811646, "sampling/importance_sampling_ratio/min": 5.044674471569341e-44, "sampling/sampling_logp_difference/max": 32.78172302246094, "sampling/sampling_logp_difference/mean": 1.115617275238037, "step": 80, "step_time": 18.73878160401364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.40625, "completions/mean_terminated_length": 16.846155166625977, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7700094953179359, "epoch": 0.00162, "frac_reward_zero_std": 0.0, "grad_norm": 0.3825273811817169, "kl": 0.3229190001729876, "learning_rate": 9.950027979886159e-06, "loss": -0.1166, "num_tokens": 3625186.0, "reward": -0.00937499850988388, "reward_std": 0.361772745847702, "rewards/rollout_reward_func/mean": -0.00937499850988388, "rewards/rollout_reward_func/std": 0.3559262156486511, "sampling/importance_sampling_ratio/max": 2.4558842182159424, "sampling/importance_sampling_ratio/mean": 0.5534465312957764, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.80569839477539, "sampling/sampling_logp_difference/mean": 1.4427781105041504, "step": 81, "step_time": 18.814884080027696 }, { "clip_ratio/high_max": 0.009310344699770212, "clip_ratio/high_mean": 0.004655172349885106, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004655172349885106, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 16.59375, "completions/mean_terminated_length": 14.592592239379883, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7749080415815115, "epoch": 0.00164, "frac_reward_zero_std": 0.0, "grad_norm": 0.3564086854457855, "kl": 0.31192881613969803, "learning_rate": 9.947787549094288e-06, "loss": -0.0314, "num_tokens": 3668341.0, "reward": 0.0031250007450580597, "reward_std": 0.3639564514160156, "rewards/rollout_reward_func/mean": 0.0031250007450580597, "rewards/rollout_reward_func/std": 0.3605411648750305, "sampling/importance_sampling_ratio/max": 2.210970640182495, "sampling/importance_sampling_ratio/mean": 0.569645345211029, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.581974029541016, "sampling/sampling_logp_difference/mean": 1.406846046447754, "step": 82, "step_time": 18.67852197101456 }, { "clip_ratio/high_max": 0.005059523973613977, "clip_ratio/high_mean": 0.0025297619868069887, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025297619868069887, "completions/clipped_ratio": 0.21875, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 17.0, "completions/mean_terminated_length": 15.75999927520752, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.8087131399661303, "epoch": 0.00166, "frac_reward_zero_std": 0.0, "grad_norm": 0.5073438286781311, "kl": 0.23526783334091306, "learning_rate": 9.945498341984929e-06, "loss": -0.0494, "num_tokens": 3712235.0, "reward": 0.04375000298023224, "reward_std": 0.32836562395095825, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.3232271671295166, "sampling/importance_sampling_ratio/max": 2.6014156341552734, "sampling/importance_sampling_ratio/mean": 0.6260418891906738, "sampling/importance_sampling_ratio/min": 2.802596928649634e-44, "sampling/sampling_logp_difference/max": 32.76388931274414, "sampling/sampling_logp_difference/mean": 1.398940086364746, "step": 83, "step_time": 20.26245708897477 }, { "clip_ratio/high_max": 0.009270833572372794, "clip_ratio/high_mean": 0.004635416786186397, "clip_ratio/low_mean": 0.0011160714784637094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005751488264650106, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 19.53125, "completions/mean_terminated_length": 16.875, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9857819750905037, "epoch": 0.00168, "frac_reward_zero_std": 0.0, "grad_norm": 0.37729546427726746, "kl": 0.2644033101387322, "learning_rate": 9.943160388754274e-06, "loss": -0.1324, "num_tokens": 3757923.0, "reward": -0.07500000298023224, "reward_std": 0.29316461086273193, "rewards/rollout_reward_func/mean": -0.07500000298023224, "rewards/rollout_reward_func/std": 0.30374014377593994, "sampling/importance_sampling_ratio/max": 2.1336076259613037, "sampling/importance_sampling_ratio/mean": 0.4085548520088196, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.882991790771484, "sampling/sampling_logp_difference/mean": 1.9943203926086426, "step": 84, "step_time": 18.827325160993496 }, { "clip_ratio/high_max": 0.005244755418971181, "clip_ratio/high_mean": 0.0026223777094855905, "clip_ratio/low_mean": 0.0018043303862214088, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004426708095706999, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 21.28125, "completions/mean_terminated_length": 19.538461685180664, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.03917414881289, "epoch": 0.0017, "frac_reward_zero_std": 0.0, "grad_norm": 0.26737290620803833, "kl": 0.636672540102154, "learning_rate": 9.940773720241503e-06, "loss": -0.0674, "num_tokens": 3803757.0, "reward": -0.012499997392296791, "reward_std": 0.35440123081207275, "rewards/rollout_reward_func/mean": -0.012499997392296791, "rewards/rollout_reward_func/std": 0.3526398241519928, "sampling/importance_sampling_ratio/max": 1.3065879344940186, "sampling/importance_sampling_ratio/mean": 0.2678338289260864, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.88888168334961, "sampling/sampling_logp_difference/mean": 2.110836982727051, "step": 85, "step_time": 19.955148267035838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0013020833721384406, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013020833721384406, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 14.40625, "completions/mean_terminated_length": 13.178571701049805, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.7874963264912367, "epoch": 0.00172, "frac_reward_zero_std": 0.0, "grad_norm": 0.3337269723415375, "kl": 0.3972016270272434, "learning_rate": 9.938338367928391e-06, "loss": -0.0637, "num_tokens": 3848162.0, "reward": 0.009375001303851604, "reward_std": 0.30323758721351624, "rewards/rollout_reward_func/mean": 0.009375001303851604, "rewards/rollout_reward_func/std": 0.2944205701351166, "sampling/importance_sampling_ratio/max": 2.433807134628296, "sampling/importance_sampling_ratio/mean": 0.5794509053230286, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.64447021484375, "sampling/sampling_logp_difference/mean": 1.6094014644622803, "step": 86, "step_time": 19.02175842100405 }, { "clip_ratio/high_max": 0.005789473652839661, "clip_ratio/high_mean": 0.0028947368264198303, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028947368264198303, "completions/clipped_ratio": 0.21875, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 17.65625, "completions/mean_terminated_length": 16.19999885559082, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8488674201071262, "epoch": 0.00174, "frac_reward_zero_std": 0.0, "grad_norm": 0.2941845655441284, "kl": 0.34437474282458425, "learning_rate": 9.935854363938876e-06, "loss": -0.0652, "num_tokens": 3893366.0, "reward": -0.03749999403953552, "reward_std": 0.36175376176834106, "rewards/rollout_reward_func/mean": -0.03749999403953552, "rewards/rollout_reward_func/std": 0.3589860200881958, "sampling/importance_sampling_ratio/max": 1.743872880935669, "sampling/importance_sampling_ratio/mean": 0.35830914974212646, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.23359298706055, "sampling/sampling_logp_difference/mean": 1.7633988857269287, "step": 87, "step_time": 20.131482289987616 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.0027791502652689815, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038567364681512117, "completions/clipped_ratio": 0.125, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.4375, "completions/mean_terminated_length": 16.035715103149414, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0161858163774014, "epoch": 0.00176, "frac_reward_zero_std": 0.0, "grad_norm": 0.4831579029560089, "kl": 0.604049801826477, "learning_rate": 9.933321741038655e-06, "loss": -0.0662, "num_tokens": 3939286.0, "reward": 0.02187499962747097, "reward_std": 0.30273666977882385, "rewards/rollout_reward_func/mean": 0.02187499962747097, "rewards/rollout_reward_func/std": 0.3034578859806061, "sampling/importance_sampling_ratio/max": 1.9579439163208008, "sampling/importance_sampling_ratio/mean": 0.38721176981925964, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.25469207763672, "sampling/sampling_logp_difference/mean": 2.2069473266601562, "step": 88, "step_time": 20.581022000013036 }, { "clip_ratio/high_max": 0.008928571827709675, "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.125, "completions/max_length": 48.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 15.875, "completions/mean_terminated_length": 14.714286804199219, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8048100117594004, "epoch": 0.00178, "frac_reward_zero_std": 0.0, "grad_norm": 0.3463306128978729, "kl": 0.5562361655756831, "learning_rate": 9.930740532634733e-06, "loss": -0.1141, "num_tokens": 3983856.0, "reward": -0.00937499850988388, "reward_std": 0.2514405846595764, "rewards/rollout_reward_func/mean": -0.00937499850988388, "rewards/rollout_reward_func/std": 0.2922210693359375, "sampling/importance_sampling_ratio/max": 2.3425495624542236, "sampling/importance_sampling_ratio/mean": 0.48476266860961914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.0360107421875, "sampling/sampling_logp_difference/mean": 1.467663049697876, "step": 89, "step_time": 18.328929315975984 }, { "clip_ratio/high_max": 0.008497807197272778, "clip_ratio/high_mean": 0.0057369989808648825, "clip_ratio/low_mean": 0.0037202382227405906, "clip_ratio/low_min": 0.0022321429569274187, "clip_ratio/region_mean": 0.009457237203605473, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 16.75, "completions/mean_terminated_length": 16.172412872314453, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0679856948554516, "epoch": 0.0018, "frac_reward_zero_std": 0.0, "grad_norm": 0.7789567708969116, "kl": 0.2981176907196641, "learning_rate": 9.928110772774995e-06, "loss": -0.0852, "num_tokens": 4028143.0, "reward": -0.009375002235174179, "reward_std": 0.28800666332244873, "rewards/rollout_reward_func/mean": -0.009375002235174179, "rewards/rollout_reward_func/std": 0.3124919533729553, "sampling/importance_sampling_ratio/max": 2.579543113708496, "sampling/importance_sampling_ratio/mean": 0.4499722123146057, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.18802261352539, "sampling/sampling_logp_difference/mean": 2.3862786293029785, "step": 90, "step_time": 19.284740295028314 }, { "clip_ratio/high_max": 0.0062500000931322575, "clip_ratio/high_mean": 0.0031250000465661287, "clip_ratio/low_mean": 0.0024074073880910873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005532407434657216, "completions/clipped_ratio": 0.125, "completions/max_length": 33.0, "completions/max_terminated_length": 33.0, "completions/mean_length": 17.03125, "completions/mean_terminated_length": 17.071430206298828, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0550036262720823, "epoch": 0.00182, "frac_reward_zero_std": 0.0, "grad_norm": 0.4539397358894348, "kl": 0.21173828886821866, "learning_rate": 9.92543249614775e-06, "loss": -0.1662, "num_tokens": 4072887.0, "reward": 0.06562499701976776, "reward_std": 0.2744126319885254, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.2835368514060974, "sampling/importance_sampling_ratio/max": 2.869591474533081, "sampling/importance_sampling_ratio/mean": 0.7010960578918457, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.915279388427734, "sampling/sampling_logp_difference/mean": 2.2113587856292725, "step": 91, "step_time": 18.90059725700121 }, { "clip_ratio/high_max": 0.008497807197272778, "clip_ratio/high_mean": 0.004248903598636389, "clip_ratio/low_mean": 0.002864583395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007113486994057894, "completions/clipped_ratio": 0.15625, "completions/max_length": 33.0, "completions/max_terminated_length": 33.0, "completions/mean_length": 17.5625, "completions/mean_terminated_length": 16.66666603088379, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9150331672281027, "epoch": 0.00184, "frac_reward_zero_std": 0.0, "grad_norm": 0.31792452931404114, "kl": 0.2721650009043515, "learning_rate": 9.922705738081279e-06, "loss": -0.1519, "num_tokens": 4118659.0, "reward": -0.08750000596046448, "reward_std": 0.32766926288604736, "rewards/rollout_reward_func/mean": -0.08750000596046448, "rewards/rollout_reward_func/std": 0.332876056432724, "sampling/importance_sampling_ratio/max": 1.8105461597442627, "sampling/importance_sampling_ratio/mean": 0.4553682804107666, "sampling/importance_sampling_ratio/min": 6.688397570222352e-42, "sampling/sampling_logp_difference/max": 31.712156295776367, "sampling/sampling_logp_difference/mean": 1.8778564929962158, "step": 92, "step_time": 19.13602161196468 }, { "clip_ratio/high_max": 0.005280172452330589, "clip_ratio/high_mean": 0.0026400862261652946, "clip_ratio/low_mean": 0.0024519230937585235, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005092009319923818, "completions/clipped_ratio": 0.15625, "completions/max_length": 48.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.9375, "completions/mean_terminated_length": 16.148147583007812, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9666279833763838, "epoch": 0.00186, "frac_reward_zero_std": 0.0, "grad_norm": 0.19049783051013947, "kl": 0.2798070516437292, "learning_rate": 9.919930534543365e-06, "loss": -0.0836, "num_tokens": 4162955.0, "reward": 0.02187500149011612, "reward_std": 0.339849591255188, "rewards/rollout_reward_func/mean": 0.02187500149011612, "rewards/rollout_reward_func/std": 0.33957695960998535, "sampling/importance_sampling_ratio/max": 2.449096918106079, "sampling/importance_sampling_ratio/mean": 0.4684128165245056, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.89448928833008, "sampling/sampling_logp_difference/mean": 1.979285478591919, "step": 93, "step_time": 19.280889026995283 }, { "clip_ratio/high_max": 0.005681818351149559, "clip_ratio/high_mean": 0.003998316824436188, "clip_ratio/low_mean": 0.0014880952658131719, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00548641209024936, "completions/clipped_ratio": 0.1875, "completions/max_length": 43.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 19.4375, "completions/mean_terminated_length": 18.0, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9105012360960245, "epoch": 0.00188, "frac_reward_zero_std": 0.0, "grad_norm": 0.42534488439559937, "kl": 0.3176561836153269, "learning_rate": 9.917106922140814e-06, "loss": -0.0619, "num_tokens": 4207311.0, "reward": 0.05000000447034836, "reward_std": 0.30478766560554504, "rewards/rollout_reward_func/mean": 0.05000000447034836, "rewards/rollout_reward_func/std": 0.2929273843765259, "sampling/importance_sampling_ratio/max": 2.641486406326294, "sampling/importance_sampling_ratio/mean": 0.5036330819129944, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.3988037109375, "sampling/sampling_logp_difference/mean": 1.8372443914413452, "step": 94, "step_time": 18.7881861939386 }, { "clip_ratio/high_max": 0.018342391354963183, "clip_ratio/high_mean": 0.009171195677481592, "clip_ratio/low_mean": 0.0010775862028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010248781880363822, "completions/clipped_ratio": 0.15625, "completions/max_length": 48.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.0625, "completions/mean_terminated_length": 16.22222328186035, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8970005530864, "epoch": 0.0019, "frac_reward_zero_std": 0.0, "grad_norm": 0.4692661762237549, "kl": 0.5785615220665932, "learning_rate": 9.914234938118988e-06, "loss": -0.1604, "num_tokens": 4251838.0, "reward": -0.012499997392296791, "reward_std": 0.3013932704925537, "rewards/rollout_reward_func/mean": -0.012499997392296791, "rewards/rollout_reward_func/std": 0.2981123626232147, "sampling/importance_sampling_ratio/max": 2.5704333782196045, "sampling/importance_sampling_ratio/mean": 0.5181933641433716, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 32.688011169433594, "sampling/sampling_logp_difference/mean": 1.565739393234253, "step": 95, "step_time": 19.649885072009056 }, { "clip_ratio/high_max": 0.0062500000931322575, "clip_ratio/high_mean": 0.0031250000465661287, "clip_ratio/low_mean": 0.01090073527302593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014025735319592059, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 16.875, "completions/mean_terminated_length": 13.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8940180093050003, "epoch": 0.00192, "frac_reward_zero_std": 0.0, "grad_norm": 0.4514530599117279, "kl": 0.646374489646405, "learning_rate": 9.911314620361294e-06, "loss": -0.0563, "num_tokens": 4296307.0, "reward": -0.02812499925494194, "reward_std": 0.3780609965324402, "rewards/rollout_reward_func/mean": -0.02812499925494194, "rewards/rollout_reward_func/std": 0.3700343072414398, "sampling/importance_sampling_ratio/max": 2.8272931575775146, "sampling/importance_sampling_ratio/mean": 0.5659167766571045, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.95698165893555, "sampling/sampling_logp_difference/mean": 1.8907653093338013, "step": 96, "step_time": 18.50336342501396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022321429569274187, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.8125, "completions/mean_terminated_length": 15.000000953674316, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.7906941995024681, "epoch": 0.00194, "frac_reward_zero_std": 0.0, "grad_norm": 0.29357045888900757, "kl": 0.36900622537359595, "learning_rate": 9.908346007388698e-06, "loss": -0.0748, "num_tokens": 4341079.0, "reward": 0.009375004097819328, "reward_std": 0.3543928861618042, "rewards/rollout_reward_func/mean": 0.009375004097819328, "rewards/rollout_reward_func/std": 0.3622148334980011, "sampling/importance_sampling_ratio/max": 1.4837974309921265, "sampling/importance_sampling_ratio/mean": 0.541071891784668, "sampling/importance_sampling_ratio/min": 1.1210387714598537e-44, "sampling/sampling_logp_difference/max": 32.92109680175781, "sampling/sampling_logp_difference/mean": 1.6148732900619507, "step": 97, "step_time": 18.94393648998812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.003705929615534842, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003705929615534842, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 16.25, "completions/mean_terminated_length": 15.60714340209961, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.901328818872571, "epoch": 0.00196, "frac_reward_zero_std": 0.0, "grad_norm": 0.8302865624427795, "kl": 0.40225035371258855, "learning_rate": 9.905329138359213e-06, "loss": -0.2494, "num_tokens": 4386431.0, "reward": -0.10312499105930328, "reward_std": 0.3195651173591614, "rewards/rollout_reward_func/mean": -0.10312499105930328, "rewards/rollout_reward_func/std": 0.311587393283844, "sampling/importance_sampling_ratio/max": 2.5116500854492188, "sampling/importance_sampling_ratio/mean": 0.644707202911377, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.88890075683594, "sampling/sampling_logp_difference/mean": 1.5290144681930542, "step": 98, "step_time": 19.34845144301653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0018382353009656072, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018382353009656072, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.40625, "completions/mean_terminated_length": 15.730770111083984, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0372680202126503, "epoch": 0.00198, "frac_reward_zero_std": 0.0, "grad_norm": 0.3332483768463135, "kl": 0.5334880028385669, "learning_rate": 9.902264053067381e-06, "loss": -0.1716, "num_tokens": 4430632.0, "reward": -0.09687501192092896, "reward_std": 0.3611540198326111, "rewards/rollout_reward_func/mean": -0.09687501192092896, "rewards/rollout_reward_func/std": 0.3551323115825653, "sampling/importance_sampling_ratio/max": 2.3234119415283203, "sampling/importance_sampling_ratio/mean": 0.5685955882072449, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.906776428222656, "sampling/sampling_logp_difference/mean": 2.0319509506225586, "step": 99, "step_time": 18.314509028976317 }, { "clip_ratio/high_max": 0.008342391345649958, "clip_ratio/high_mean": 0.004171195672824979, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004171195672824979, "completions/clipped_ratio": 0.125, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 18.71875, "completions/mean_terminated_length": 17.5, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0023212619125843, "epoch": 0.002, "frac_reward_zero_std": 0.0, "grad_norm": 0.4161045551300049, "kl": 0.29766706423833966, "learning_rate": 9.899150791943747e-06, "loss": -0.1421, "num_tokens": 4475371.0, "reward": -0.046875, "reward_std": 0.39755111932754517, "rewards/rollout_reward_func/mean": -0.046875, "rewards/rollout_reward_func/std": 0.3943016529083252, "sampling/importance_sampling_ratio/max": 1.8461623191833496, "sampling/importance_sampling_ratio/mean": 0.4691755473613739, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.796085357666016, "sampling/sampling_logp_difference/mean": 2.2076704502105713, "step": 100, "step_time": 20.416418327004067 }, { "clip_ratio/high_max": 0.007424242561683059, "clip_ratio/high_mean": 0.0037121212808415294, "clip_ratio/low_mean": 0.01964962133206427, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.023361742729321122, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 25.0, "completions/mean_length": 15.40625, "completions/mean_terminated_length": 12.923077583312988, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8084716349840164, "epoch": 0.00202, "frac_reward_zero_std": 0.0, "grad_norm": 0.6315691471099854, "kl": 0.6543253883719444, "learning_rate": 9.895989396054334e-06, "loss": -0.1667, "num_tokens": 4520777.0, "reward": -0.03437500074505806, "reward_std": 0.3163359463214874, "rewards/rollout_reward_func/mean": -0.03437500074505806, "rewards/rollout_reward_func/std": 0.30754950642585754, "sampling/importance_sampling_ratio/max": 2.903498649597168, "sampling/importance_sampling_ratio/mean": 0.7111390829086304, "sampling/importance_sampling_ratio/min": 5.605193857299268e-44, "sampling/sampling_logp_difference/max": 32.54340362548828, "sampling/sampling_logp_difference/mean": 1.8587098121643066, "step": 101, "step_time": 18.644144884005073 }, { "clip_ratio/high_max": 0.018601190531626344, "clip_ratio/high_mean": 0.010550595005042851, "clip_ratio/low_mean": 0.004744112433400005, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015294707438442856, "completions/clipped_ratio": 0.15625, "completions/max_length": 40.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.6875, "completions/mean_terminated_length": 15.481481552124023, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.065512353554368, "epoch": 0.00204, "frac_reward_zero_std": 0.0, "grad_norm": 0.3798440098762512, "kl": 0.36983325285837054, "learning_rate": 9.892779907100084e-06, "loss": -0.1467, "num_tokens": 4566529.0, "reward": -0.0468750037252903, "reward_std": 0.32145947217941284, "rewards/rollout_reward_func/mean": -0.0468750037252903, "rewards/rollout_reward_func/std": 0.3350343108177185, "sampling/importance_sampling_ratio/max": 1.5625205039978027, "sampling/importance_sampling_ratio/mean": 0.384440153837204, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.076053619384766, "sampling/sampling_logp_difference/mean": 2.3630623817443848, "step": 102, "step_time": 19.2536566619965 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.002327586174942553, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003577586147002876, "completions/clipped_ratio": 0.15625, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 17.78125, "completions/mean_terminated_length": 15.851852416992188, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9839259628206491, "epoch": 0.00206, "frac_reward_zero_std": 0.0, "grad_norm": 0.4616515338420868, "kl": 0.7510647755116224, "learning_rate": 9.889522367416332e-06, "loss": -0.0834, "num_tokens": 4610099.0, "reward": -0.01249999925494194, "reward_std": 0.3659454882144928, "rewards/rollout_reward_func/mean": -0.01249999925494194, "rewards/rollout_reward_func/std": 0.3866731524467468, "sampling/importance_sampling_ratio/max": 1.761243462562561, "sampling/importance_sampling_ratio/mean": 0.3883262872695923, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.83694076538086, "sampling/sampling_logp_difference/mean": 2.0304391384124756, "step": 103, "step_time": 18.906788395019248 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0027153200935572386, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004668445093557239, "completions/clipped_ratio": 0.1875, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 17.09375, "completions/mean_terminated_length": 14.807692527770996, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.8971188347786665, "epoch": 0.00208, "frac_reward_zero_std": 0.0, "grad_norm": 0.28598764538764954, "kl": 0.38059982378035784, "learning_rate": 9.886216819972226e-06, "loss": -0.1519, "num_tokens": 4654837.0, "reward": -0.012499995529651642, "reward_std": 0.34107545018196106, "rewards/rollout_reward_func/mean": -0.012499995529651642, "rewards/rollout_reward_func/std": 0.34617727994918823, "sampling/importance_sampling_ratio/max": 2.2791595458984375, "sampling/importance_sampling_ratio/mean": 0.5831276178359985, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.04011535644531, "sampling/sampling_logp_difference/mean": 1.798982858657837, "step": 104, "step_time": 19.595527348021278 }, { "clip_ratio/high_max": 0.004546957788988948, "clip_ratio/high_mean": 0.002273478894494474, "clip_ratio/low_mean": 0.0034053156850859523, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005678794579580426, "completions/clipped_ratio": 0.09375, "completions/max_length": 43.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 18.625, "completions/mean_terminated_length": 17.86206817626953, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.115008618682623, "epoch": 0.0021, "frac_reward_zero_std": 0.0, "grad_norm": 0.20098000764846802, "kl": 0.5672413087449968, "learning_rate": 9.882863308370175e-06, "loss": -0.0782, "num_tokens": 4700263.0, "reward": -0.04999999701976776, "reward_std": 0.3395112156867981, "rewards/rollout_reward_func/mean": -0.04999999701976776, "rewards/rollout_reward_func/std": 0.3321484327316284, "sampling/importance_sampling_ratio/max": 1.7006779909133911, "sampling/importance_sampling_ratio/mean": 0.39403340220451355, "sampling/importance_sampling_ratio/min": 1.961817850054744e-44, "sampling/sampling_logp_difference/max": 33.54515075683594, "sampling/sampling_logp_difference/mean": 2.4288644790649414, "step": 105, "step_time": 19.4401298360317 }, { "clip_ratio/high_max": 0.01066246721893549, "clip_ratio/high_mean": 0.005331233609467745, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0066899292869493365, "completions/clipped_ratio": 0.09375, "completions/max_length": 31.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.25, "completions/mean_terminated_length": 17.379310607910156, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0723892785608768, "epoch": 0.00212, "frac_reward_zero_std": 0.0, "grad_norm": 0.32190030813217163, "kl": 0.43250425485894084, "learning_rate": 9.879461876845255e-06, "loss": 0.0395, "num_tokens": 4745262.0, "reward": 0.02812500111758709, "reward_std": 0.2911275029182434, "rewards/rollout_reward_func/mean": 0.02812500111758709, "rewards/rollout_reward_func/std": 0.27967074513435364, "sampling/importance_sampling_ratio/max": 2.2287676334381104, "sampling/importance_sampling_ratio/mean": 0.4014021158218384, "sampling/importance_sampling_ratio/min": 7.132609183413319e-43, "sampling/sampling_logp_difference/max": 32.395416259765625, "sampling/sampling_logp_difference/mean": 2.1177737712860107, "step": 106, "step_time": 17.986823407030897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 43.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 18.375, "completions/mean_terminated_length": 16.625, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9932642411440611, "epoch": 0.00214, "frac_reward_zero_std": 0.0, "grad_norm": 0.3178599774837494, "kl": 0.21847084793262184, "learning_rate": 9.876012570264658e-06, "loss": -0.1727, "num_tokens": 4790122.0, "reward": -0.04062499478459358, "reward_std": 0.3485024571418762, "rewards/rollout_reward_func/mean": -0.04062499478459358, "rewards/rollout_reward_func/std": 0.3425326943397522, "sampling/importance_sampling_ratio/max": 2.4724929332733154, "sampling/importance_sampling_ratio/mean": 0.5137382745742798, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.879791259765625, "sampling/sampling_logp_difference/mean": 2.009403705596924, "step": 107, "step_time": 19.228616459004115 }, { "clip_ratio/high_max": 0.00763136288151145, "clip_ratio/high_mean": 0.003815681440755725, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003815681440755725, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 17.0625, "completions/mean_terminated_length": 14.84615421295166, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9192055761814117, "epoch": 0.00216, "frac_reward_zero_std": 0.0, "grad_norm": 0.3864194452762604, "kl": 0.6165243466384709, "learning_rate": 9.872515434127064e-06, "loss": -0.1469, "num_tokens": 4835086.0, "reward": -0.028124995529651642, "reward_std": 0.2870478928089142, "rewards/rollout_reward_func/mean": -0.028124995529651642, "rewards/rollout_reward_func/std": 0.29428356885910034, "sampling/importance_sampling_ratio/max": 2.183530569076538, "sampling/importance_sampling_ratio/mean": 0.4282582998275757, "sampling/importance_sampling_ratio/min": 2.98756832594051e-42, "sampling/sampling_logp_difference/max": 33.28175735473633, "sampling/sampling_logp_difference/mean": 1.585343837738037, "step": 108, "step_time": 18.599305607989663 }, { "clip_ratio/high_max": 0.010529891354963183, "clip_ratio/high_mean": 0.005264945677481592, "clip_ratio/low_mean": 0.005862856633029878, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01112780231051147, "completions/clipped_ratio": 0.125, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 16.03125, "completions/mean_terminated_length": 15.750000953674316, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.055977139621973, "epoch": 0.00218, "frac_reward_zero_std": 0.0, "grad_norm": 0.3547895848751068, "kl": 0.43764252681285143, "learning_rate": 9.86897051456206e-06, "loss": -0.1742, "num_tokens": 4880285.0, "reward": -0.03437500447034836, "reward_std": 0.3050459921360016, "rewards/rollout_reward_func/mean": -0.03437500447034836, "rewards/rollout_reward_func/std": 0.30964013934135437, "sampling/importance_sampling_ratio/max": 2.3697941303253174, "sampling/importance_sampling_ratio/mean": 0.5352022051811218, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.680580139160156, "sampling/sampling_logp_difference/mean": 2.418346405029297, "step": 109, "step_time": 19.00303660800273 }, { "clip_ratio/high_max": 0.015252976445481181, "clip_ratio/high_mean": 0.007626488222740591, "clip_ratio/low_mean": 0.004758064402267337, "clip_ratio/low_min": 0.0024999999441206455, "clip_ratio/region_mean": 0.012384552625007927, "completions/clipped_ratio": 0.21875, "completions/max_length": 44.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 20.09375, "completions/mean_terminated_length": 18.03999900817871, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.168134031817317, "epoch": 0.0022, "frac_reward_zero_std": 0.0, "grad_norm": 0.33960315585136414, "kl": 0.23329190304502845, "learning_rate": 9.865377858329539e-06, "loss": -0.1415, "num_tokens": 4926436.0, "reward": -0.17500001192092896, "reward_std": 0.32490813732147217, "rewards/rollout_reward_func/mean": -0.17500001192092896, "rewards/rollout_reward_func/std": 0.33020034432411194, "sampling/importance_sampling_ratio/max": 2.7076408863067627, "sampling/importance_sampling_ratio/mean": 0.2977999448776245, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.52927780151367, "sampling/sampling_logp_difference/mean": 2.462986946105957, "step": 110, "step_time": 19.656628399025067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 16.65625, "completions/mean_terminated_length": 15.333333015441895, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8673884384334087, "epoch": 0.00222, "frac_reward_zero_std": 0.0, "grad_norm": 0.2415434718132019, "kl": 0.22600022330880165, "learning_rate": 9.861737512819058e-06, "loss": -0.1408, "num_tokens": 4970456.0, "reward": 0.06562499701976776, "reward_std": 0.2698144316673279, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.2634870111942291, "sampling/importance_sampling_ratio/max": 2.4170124530792236, "sampling/importance_sampling_ratio/mean": 0.6084898114204407, "sampling/importance_sampling_ratio/min": 5.184804318001823e-44, "sampling/sampling_logp_difference/max": 32.73359680175781, "sampling/sampling_logp_difference/mean": 1.6610054969787598, "step": 111, "step_time": 17.66978601401206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0015625000232830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015625000232830644, "completions/clipped_ratio": 0.03125, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 14.5625, "completions/mean_terminated_length": 14.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7942358087748289, "epoch": 0.00224, "frac_reward_zero_std": 0.0, "grad_norm": 0.29310905933380127, "kl": 0.2907741190865636, "learning_rate": 9.858049526049241e-06, "loss": -0.1328, "num_tokens": 5013769.0, "reward": 0.10000000149011612, "reward_std": 0.2583029866218567, "rewards/rollout_reward_func/mean": 0.10000000149011612, "rewards/rollout_reward_func/std": 0.2602728009223938, "sampling/importance_sampling_ratio/max": 2.415384292602539, "sampling/importance_sampling_ratio/mean": 0.6703503727912903, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.99806213378906, "sampling/sampling_logp_difference/mean": 1.5757017135620117, "step": 112, "step_time": 17.650229931037757 }, { "clip_ratio/high_max": 0.0013586956774815917, "clip_ratio/high_mean": 0.0006793478387407959, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019293478108011186, "completions/clipped_ratio": 0.15625, "completions/max_length": 46.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.59375, "completions/mean_terminated_length": 16.185184478759766, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0455231219530106, "epoch": 0.00226, "frac_reward_zero_std": 0.0, "grad_norm": 0.3558836877346039, "kl": 1.0085226274095476, "learning_rate": 9.85431394666712e-06, "loss": -0.1875, "num_tokens": 5059780.0, "reward": -0.16249999403953552, "reward_std": 0.30503323674201965, "rewards/rollout_reward_func/mean": -0.16249999403953552, "rewards/rollout_reward_func/std": 0.30664682388305664, "sampling/importance_sampling_ratio/max": 2.0053913593292236, "sampling/importance_sampling_ratio/mean": 0.4424915313720703, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.84424591064453, "sampling/sampling_logp_difference/mean": 2.1307857036590576, "step": 113, "step_time": 18.748474471984082 }, { "clip_ratio/high_max": 0.026622807839885354, "clip_ratio/high_mean": 0.014427475514821708, "clip_ratio/low_mean": 0.0032402074430137873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017667682957835495, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.0, "completions/mean_terminated_length": 15.375, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.1436996273696423, "epoch": 0.00228, "frac_reward_zero_std": 0.0, "grad_norm": 0.27033093571662903, "kl": 0.22517262352630496, "learning_rate": 9.850530823947519e-06, "loss": -0.0898, "num_tokens": 5104073.0, "reward": 0.02500000223517418, "reward_std": 0.31351253390312195, "rewards/rollout_reward_func/mean": 0.02500000223517418, "rewards/rollout_reward_func/std": 0.31004682183265686, "sampling/importance_sampling_ratio/max": 1.8097002506256104, "sampling/importance_sampling_ratio/mean": 0.40496599674224854, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.53898239135742, "sampling/sampling_logp_difference/mean": 2.3795530796051025, "step": 114, "step_time": 19.33433389100537 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0017671131645329297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029245205805636942, "completions/clipped_ratio": 0.34375, "completions/max_length": 48.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.875, "completions/mean_terminated_length": 14.380952835083008, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.9559607747942209, "epoch": 0.0023, "frac_reward_zero_std": 0.0, "grad_norm": 0.22825396060943604, "kl": 0.3186634569428861, "learning_rate": 9.846700207792381e-06, "loss": -0.1224, "num_tokens": 5148614.0, "reward": -0.08749999850988388, "reward_std": 0.38465750217437744, "rewards/rollout_reward_func/mean": -0.08749999850988388, "rewards/rollout_reward_func/std": 0.3850010633468628, "sampling/importance_sampling_ratio/max": 2.0010852813720703, "sampling/importance_sampling_ratio/mean": 0.5118167996406555, "sampling/importance_sampling_ratio/min": 1.0299543712787405e-42, "sampling/sampling_logp_difference/max": 32.216793060302734, "sampling/sampling_logp_difference/mean": 1.73093581199646, "step": 115, "step_time": 19.24643052103056 }, { "clip_ratio/high_max": 0.020863970974460244, "clip_ratio/high_mean": 0.010431985487230122, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010431985487230122, "completions/clipped_ratio": 0.15625, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 16.375, "completions/mean_terminated_length": 14.666666984558105, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8623074479401112, "epoch": 0.00232, "frac_reward_zero_std": 0.0, "grad_norm": 0.16300880908966064, "kl": 0.36015377612784505, "learning_rate": 9.842822148730122e-06, "loss": -0.096, "num_tokens": 5192980.0, "reward": 0.04375000298023224, "reward_std": 0.32816892862319946, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.3181980550289154, "sampling/importance_sampling_ratio/max": 2.114192247390747, "sampling/importance_sampling_ratio/mean": 0.5595809817314148, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.32615661621094, "sampling/sampling_logp_difference/mean": 1.6640102863311768, "step": 116, "step_time": 18.560153916012496 }, { "clip_ratio/high_max": 0.004807692486792803, "clip_ratio/high_mean": 0.0024038462433964014, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024038462433964014, "completions/clipped_ratio": 0.1875, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 18.25, "completions/mean_terminated_length": 17.846155166625977, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9398874640464783, "epoch": 0.00234, "frac_reward_zero_std": 0.0, "grad_norm": 0.2817807197570801, "kl": 0.2598775648511946, "learning_rate": 9.838896697914964e-06, "loss": -0.0648, "num_tokens": 5236468.0, "reward": 0.006250003352761269, "reward_std": 0.30668550729751587, "rewards/rollout_reward_func/mean": 0.006250003352761269, "rewards/rollout_reward_func/std": 0.3232271671295166, "sampling/importance_sampling_ratio/max": 2.0378000736236572, "sampling/importance_sampling_ratio/mean": 0.5354421138763428, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.52336883544922, "sampling/sampling_logp_difference/mean": 1.8985798358917236, "step": 117, "step_time": 18.974892347992864 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0024671052815392613, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004420230281539261, "completions/clipped_ratio": 0.1875, "completions/max_length": 38.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.21875, "completions/mean_terminated_length": 14.84615421295166, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.022923443466425, "epoch": 0.00236, "frac_reward_zero_std": 0.0, "grad_norm": 0.30295687913894653, "kl": 0.3458759873174131, "learning_rate": 9.834923907126256e-06, "loss": -0.1287, "num_tokens": 5281876.0, "reward": -0.09687499701976776, "reward_std": 0.3245706558227539, "rewards/rollout_reward_func/mean": -0.09687499701976776, "rewards/rollout_reward_func/std": 0.3306731879711151, "sampling/importance_sampling_ratio/max": 2.077159881591797, "sampling/importance_sampling_ratio/mean": 0.4992758631706238, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.98240280151367, "sampling/sampling_logp_difference/mean": 1.9145290851593018, "step": 118, "step_time": 19.129537480024737 }, { "clip_ratio/high_max": 0.0026041667442768812, "clip_ratio/high_mean": 0.0013020833721384406, "clip_ratio/low_mean": 0.0038567364681512117, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005158819840289652, "completions/clipped_ratio": 0.1875, "completions/max_length": 30.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 15.90625, "completions/mean_terminated_length": 14.923077583312988, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.9174267090857029, "epoch": 0.00238, "frac_reward_zero_std": 0.0, "grad_norm": 2.2251033782958984, "kl": 3.181407097610645, "learning_rate": 9.830903828767796e-06, "loss": -0.1071, "num_tokens": 5325205.0, "reward": -9.313225746154785e-10, "reward_std": 0.3049643635749817, "rewards/rollout_reward_func/mean": -9.313225746154785e-10, "rewards/rollout_reward_func/std": 0.29621264338493347, "sampling/importance_sampling_ratio/max": 2.9677541255950928, "sampling/importance_sampling_ratio/mean": 0.702781081199646, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.99253463745117, "sampling/sampling_logp_difference/mean": 2.0048320293426514, "step": 119, "step_time": 18.39210702200944 }, { "clip_ratio/high_max": 0.005121237598359585, "clip_ratio/high_mean": 0.0025606187991797924, "clip_ratio/low_mean": 0.0025778620038181543, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005138480802997947, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 15.84375, "completions/mean_terminated_length": 14.59999942779541, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9609023258090019, "epoch": 0.0024, "frac_reward_zero_std": 0.0, "grad_norm": 0.22599996626377106, "kl": 0.604760923422873, "learning_rate": 9.826836515867131e-06, "loss": -0.0685, "num_tokens": 5370550.0, "reward": -0.05312499776482582, "reward_std": 0.318830281496048, "rewards/rollout_reward_func/mean": -0.05312499776482582, "rewards/rollout_reward_func/std": 0.31003057956695557, "sampling/importance_sampling_ratio/max": 2.027418613433838, "sampling/importance_sampling_ratio/mean": 0.37864604592323303, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.92014694213867, "sampling/sampling_logp_difference/mean": 1.7322845458984375, "step": 120, "step_time": 18.77507773200341 }, { "clip_ratio/high_max": 0.004166666883975267, "clip_ratio/high_mean": 0.0020833334419876337, "clip_ratio/low_mean": 0.003906250116415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0059895835584029555, "completions/clipped_ratio": 0.1875, "completions/max_length": 43.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.65625, "completions/mean_terminated_length": 15.230770111083984, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9329271856695414, "epoch": 0.00242, "frac_reward_zero_std": 0.0, "grad_norm": 0.22902590036392212, "kl": 0.3763399440795183, "learning_rate": 9.822722022074871e-06, "loss": -0.0585, "num_tokens": 5415425.0, "reward": -0.056249991059303284, "reward_std": 0.30360814929008484, "rewards/rollout_reward_func/mean": -0.056249991059303284, "rewards/rollout_reward_func/std": 0.33690956234931946, "sampling/importance_sampling_ratio/max": 1.9401575326919556, "sampling/importance_sampling_ratio/mean": 0.38801509141921997, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.435123443603516, "sampling/sampling_logp_difference/mean": 2.0654385089874268, "step": 121, "step_time": 19.29055810799764 }, { "clip_ratio/high_max": 0.023074534256011248, "clip_ratio/high_mean": 0.011537267128005624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011537267128005624, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 19.71875, "completions/mean_terminated_length": 16.88461685180664, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9773456826806068, "epoch": 0.00244, "frac_reward_zero_std": 0.0, "grad_norm": 0.2527378797531128, "kl": 0.5448608095757663, "learning_rate": 9.818560401663972e-06, "loss": -0.0383, "num_tokens": 5460823.0, "reward": -0.04062499478459358, "reward_std": 0.3731716275215149, "rewards/rollout_reward_func/mean": -0.04062499478459358, "rewards/rollout_reward_func/std": 0.3723156750202179, "sampling/importance_sampling_ratio/max": 1.7969542741775513, "sampling/importance_sampling_ratio/mean": 0.3104862868785858, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.681026458740234, "sampling/sampling_logp_difference/mean": 1.9853816032409668, "step": 122, "step_time": 19.55073093395913 }, { "clip_ratio/high_max": 0.012130231130868196, "clip_ratio/high_mean": 0.006065115565434098, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006065115565434098, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 14.59375, "completions/mean_terminated_length": 13.038461685180664, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.8461234699934721, "epoch": 0.00246, "frac_reward_zero_std": 0.0, "grad_norm": 0.23689062893390656, "kl": 0.29490388091653585, "learning_rate": 9.814351709529018e-06, "loss": -0.081, "num_tokens": 5505029.0, "reward": -0.0031250007450580597, "reward_std": 0.30784663558006287, "rewards/rollout_reward_func/mean": -0.0031250007450580597, "rewards/rollout_reward_func/std": 0.29890596866607666, "sampling/importance_sampling_ratio/max": 2.707200050354004, "sampling/importance_sampling_ratio/mean": 0.5793750286102295, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.656742095947266, "sampling/sampling_logp_difference/mean": 1.7221602201461792, "step": 123, "step_time": 17.369714328990085 }, { "clip_ratio/high_max": 0.007156385574489832, "clip_ratio/high_mean": 0.003578192787244916, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003578192787244916, "completions/clipped_ratio": 0.15625, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 16.4375, "completions/mean_terminated_length": 16.037036895751953, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0766904633492231, "epoch": 0.00248, "frac_reward_zero_std": 0.0, "grad_norm": 0.3088851571083069, "kl": 0.6402361567597836, "learning_rate": 9.8100960011855e-06, "loss": -0.0683, "num_tokens": 5549414.0, "reward": -0.06875000149011612, "reward_std": 0.32725316286087036, "rewards/rollout_reward_func/mean": -0.06875000149011612, "rewards/rollout_reward_func/std": 0.31769075989723206, "sampling/importance_sampling_ratio/max": 1.9479858875274658, "sampling/importance_sampling_ratio/mean": 0.4424622654914856, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.81730270385742, "sampling/sampling_logp_difference/mean": 2.1119132041931152, "step": 124, "step_time": 18.462439519993495 }, { "clip_ratio/high_max": 0.005759804043918848, "clip_ratio/high_mean": 0.002879902021959424, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002879902021959424, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.6875, "completions/mean_terminated_length": 16.31999969482422, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.788985189050436, "epoch": 0.0025, "frac_reward_zero_std": 0.0, "grad_norm": 0.2477109432220459, "kl": 0.6310231569223106, "learning_rate": 9.805793332769095e-06, "loss": -0.1283, "num_tokens": 5594252.0, "reward": -0.006249997764825821, "reward_std": 0.3313668370246887, "rewards/rollout_reward_func/mean": -0.006249997764825821, "rewards/rollout_reward_func/std": 0.33498916029930115, "sampling/importance_sampling_ratio/max": 2.131263256072998, "sampling/importance_sampling_ratio/mean": 0.5570611953735352, "sampling/importance_sampling_ratio/min": 7.636235851491658e-41, "sampling/sampling_logp_difference/max": 32.223636627197266, "sampling/sampling_logp_difference/mean": 1.262578010559082, "step": 125, "step_time": 19.565605228985078 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0018382353009656072, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018382353009656072, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 15.59375, "completions/mean_terminated_length": 13.615385055541992, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.921806002035737, "epoch": 0.00252, "frac_reward_zero_std": 0.0, "grad_norm": 0.27717509865760803, "kl": 0.3032499118708074, "learning_rate": 9.801443761034903e-06, "loss": -0.1171, "num_tokens": 5638788.0, "reward": -0.05312500149011612, "reward_std": 0.3077625036239624, "rewards/rollout_reward_func/mean": -0.05312500149011612, "rewards/rollout_reward_func/std": 0.29836583137512207, "sampling/importance_sampling_ratio/max": 2.1805126667022705, "sampling/importance_sampling_ratio/mean": 0.6748624444007874, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.74352264404297, "sampling/sampling_logp_difference/mean": 1.8188767433166504, "step": 126, "step_time": 18.29811745500774 }, { "clip_ratio/high_max": 0.0060068650636821985, "clip_ratio/high_mean": 0.0030034325318410993, "clip_ratio/low_mean": 0.0006944444612599909, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00369787699310109, "completions/clipped_ratio": 0.25, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 18.15625, "completions/mean_terminated_length": 17.541667938232422, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9747796934098005, "epoch": 0.00254, "frac_reward_zero_std": 0.0, "grad_norm": 0.4400172829627991, "kl": 0.20168441953137517, "learning_rate": 9.797047343356718e-06, "loss": 0.0451, "num_tokens": 5683553.0, "reward": 0.02187499962747097, "reward_std": 0.32789695262908936, "rewards/rollout_reward_func/mean": 0.02187499962747097, "rewards/rollout_reward_func/std": 0.3299407362937927, "sampling/importance_sampling_ratio/max": 2.76515793800354, "sampling/importance_sampling_ratio/mean": 0.5330810546875, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.5863037109375, "sampling/sampling_logp_difference/mean": 1.630990982055664, "step": 127, "step_time": 18.721223034954164 }, { "clip_ratio/high_max": 0.009535845601931214, "clip_ratio/high_mean": 0.005744485300965607, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00699448527302593, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.375, "completions/mean_terminated_length": 16.521739959716797, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.0891878362745047, "epoch": 0.00256, "frac_reward_zero_std": 0.0, "grad_norm": 0.21110497415065765, "kl": 0.3828792478889227, "learning_rate": 9.792604137726259e-06, "loss": -0.0877, "num_tokens": 5729385.0, "reward": -0.05624999850988388, "reward_std": 0.327867329120636, "rewards/rollout_reward_func/mean": -0.05624999850988388, "rewards/rollout_reward_func/std": 0.3573536276817322, "sampling/importance_sampling_ratio/max": 2.2357826232910156, "sampling/importance_sampling_ratio/mean": 0.3401555120944977, "sampling/importance_sampling_ratio/min": 7.987401246651457e-44, "sampling/sampling_logp_difference/max": 33.68578338623047, "sampling/sampling_logp_difference/mean": 1.9078372716903687, "step": 128, "step_time": 18.728147229980095 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.003724767128005624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003724767128005624, "completions/clipped_ratio": 0.125, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 16.53125, "completions/mean_terminated_length": 16.60714340209961, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.013078429736197, "epoch": 0.00258, "frac_reward_zero_std": 0.0, "grad_norm": 0.34069105982780457, "kl": 0.2475765929557383, "learning_rate": 9.788114202752415e-06, "loss": -0.1464, "num_tokens": 5773551.0, "reward": -0.10000000894069672, "reward_std": 0.3381536602973938, "rewards/rollout_reward_func/mean": -0.10000000894069672, "rewards/rollout_reward_func/std": 0.32527902722358704, "sampling/importance_sampling_ratio/max": 1.968001127243042, "sampling/importance_sampling_ratio/mean": 0.40390270948410034, "sampling/importance_sampling_ratio/min": 6.6001157669698884e-43, "sampling/sampling_logp_difference/max": 33.27931213378906, "sampling/sampling_logp_difference/mean": 1.890998125076294, "step": 129, "step_time": 18.924951983965002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 46.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 17.75, "completions/mean_terminated_length": 15.727272987365723, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.006397681310773, "epoch": 0.0026, "frac_reward_zero_std": 0.0, "grad_norm": 0.30860334634780884, "kl": 0.4598256992176175, "learning_rate": 9.78357759766046e-06, "loss": -0.1224, "num_tokens": 5817351.0, "reward": -0.12187499552965164, "reward_std": 0.33498114347457886, "rewards/rollout_reward_func/mean": -0.12187499552965164, "rewards/rollout_reward_func/std": 0.3414715528488159, "sampling/importance_sampling_ratio/max": 1.840442180633545, "sampling/importance_sampling_ratio/mean": 0.44890153408050537, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.73358917236328, "sampling/sampling_logp_difference/mean": 1.5958611965179443, "step": 130, "step_time": 19.40002142895537 }, { "clip_ratio/high_max": 0.008676470490172505, "clip_ratio/high_mean": 0.004338235245086253, "clip_ratio/low_mean": 0.009062499972060323, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013400735217146575, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 21.65625, "completions/mean_terminated_length": 19.38461685180664, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1306105609983206, "epoch": 0.00262, "frac_reward_zero_std": 0.0, "grad_norm": 0.06975017488002777, "kl": 0.34205463365651667, "learning_rate": 9.778994382291283e-06, "loss": 0.0084, "num_tokens": 5864996.0, "reward": -0.14999999105930328, "reward_std": 0.3423662483692169, "rewards/rollout_reward_func/mean": -0.14999999105930328, "rewards/rollout_reward_func/std": 0.36367300152778625, "sampling/importance_sampling_ratio/max": 1.2536081075668335, "sampling/importance_sampling_ratio/mean": 0.16064339876174927, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.461307525634766, "sampling/sampling_logp_difference/mean": 2.145761489868164, "step": 131, "step_time": 20.674202478010557 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.34375, "completions/max_length": 40.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 18.28125, "completions/mean_terminated_length": 14.809524536132812, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9515143316239119, "epoch": 0.00264, "frac_reward_zero_std": 0.0, "grad_norm": 0.36660152673721313, "kl": 0.1511614341288805, "learning_rate": 9.774364617100596e-06, "loss": -0.1257, "num_tokens": 5909429.0, "reward": -0.04374999552965164, "reward_std": 0.341938853263855, "rewards/rollout_reward_func/mean": -0.04374999552965164, "rewards/rollout_reward_func/std": 0.36094632744789124, "sampling/importance_sampling_ratio/max": 1.9729372262954712, "sampling/importance_sampling_ratio/mean": 0.5636782646179199, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.087276458740234, "sampling/sampling_logp_difference/mean": 1.5016731023788452, "step": 132, "step_time": 19.00610834294639 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 16.8125, "completions/mean_terminated_length": 15.481481552124023, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.8940423242747784, "epoch": 0.00266, "frac_reward_zero_std": 0.0, "grad_norm": 0.5627111196517944, "kl": 0.26651127939112484, "learning_rate": 9.769688363158127e-06, "loss": -0.1864, "num_tokens": 5954067.0, "reward": 0.02812500298023224, "reward_std": 0.3080267906188965, "rewards/rollout_reward_func/mean": 0.02812500298023224, "rewards/rollout_reward_func/std": 0.30504825711250305, "sampling/importance_sampling_ratio/max": 2.169422149658203, "sampling/importance_sampling_ratio/mean": 0.6638503074645996, "sampling/importance_sampling_ratio/min": 1.634579626173291e-39, "sampling/sampling_logp_difference/max": 33.07734680175781, "sampling/sampling_logp_difference/mean": 1.4136009216308594, "step": 133, "step_time": 18.793646427016938 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.01118607958778739, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012544774916023016, "completions/clipped_ratio": 0.1875, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 16.28125, "completions/mean_terminated_length": 15.19230842590332, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.949059996753931, "epoch": 0.00268, "frac_reward_zero_std": 0.0, "grad_norm": 0.283564954996109, "kl": 0.7089119637385011, "learning_rate": 9.76496568214683e-06, "loss": -0.1564, "num_tokens": 5998417.0, "reward": -0.0624999962747097, "reward_std": 0.3394075036048889, "rewards/rollout_reward_func/mean": -0.0624999962747097, "rewards/rollout_reward_func/std": 0.34617727994918823, "sampling/importance_sampling_ratio/max": 2.3371710777282715, "sampling/importance_sampling_ratio/mean": 0.6671672463417053, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.431026458740234, "sampling/sampling_logp_difference/mean": 1.546385645866394, "step": 134, "step_time": 18.502708706000703 }, { "clip_ratio/high_max": 0.010297049768269062, "clip_ratio/high_mean": 0.005148524884134531, "clip_ratio/low_mean": 0.0011160714784637094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00626459636259824, "completions/clipped_ratio": 0.09375, "completions/max_length": 48.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 20.96875, "completions/mean_terminated_length": 20.379310607910156, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1371056474745274, "epoch": 0.0027, "frac_reward_zero_std": 0.0, "grad_norm": 0.21995091438293457, "kl": 0.21290292288176715, "learning_rate": 9.760196636362058e-06, "loss": -0.158, "num_tokens": 6043991.0, "reward": 0.0312500037252903, "reward_std": 0.2981363534927368, "rewards/rollout_reward_func/mean": 0.0312500037252903, "rewards/rollout_reward_func/std": 0.32372578978538513, "sampling/importance_sampling_ratio/max": 1.9129620790481567, "sampling/importance_sampling_ratio/mean": 0.3123532235622406, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.6076545715332, "sampling/sampling_logp_difference/mean": 2.475574493408203, "step": 135, "step_time": 19.73143680000794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0076190283871255815, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0076190283871255815, "completions/clipped_ratio": 0.1875, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 19.5, "completions/mean_terminated_length": 17.230770111083984, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0226788837462664, "epoch": 0.00272, "frac_reward_zero_std": 0.0, "grad_norm": 0.46455520391464233, "kl": 0.1643164176493883, "learning_rate": 9.755381288710751e-06, "loss": -0.1894, "num_tokens": 6088062.0, "reward": -0.04375000298023224, "reward_std": 0.29718756675720215, "rewards/rollout_reward_func/mean": -0.04375000298023224, "rewards/rollout_reward_func/std": 0.3015417158603668, "sampling/importance_sampling_ratio/max": 2.0267233848571777, "sampling/importance_sampling_ratio/mean": 0.45081472396850586, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.28885269165039, "sampling/sampling_logp_difference/mean": 1.953803300857544, "step": 136, "step_time": 19.454062949997024 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.0017989309271797538, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032193855149671435, "completions/clipped_ratio": 0.15625, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 17.75, "completions/mean_terminated_length": 17.814815521240234, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0006270799785852, "epoch": 0.00274, "frac_reward_zero_std": 0.0, "grad_norm": 0.23055341839790344, "kl": 0.18852156505454332, "learning_rate": 9.750519702710598e-06, "loss": -0.1419, "num_tokens": 6132503.0, "reward": -0.04374999925494194, "reward_std": 0.33054372668266296, "rewards/rollout_reward_func/mean": -0.04374999925494194, "rewards/rollout_reward_func/std": 0.325217068195343, "sampling/importance_sampling_ratio/max": 2.0637221336364746, "sampling/importance_sampling_ratio/mean": 0.5583310127258301, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.27931213378906, "sampling/sampling_logp_difference/mean": 2.184098720550537, "step": 137, "step_time": 19.350657253002282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0021130952518433332, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021130952518433332, "completions/clipped_ratio": 0.1875, "completions/max_length": 50.0, "completions/max_terminated_length": 50.0, "completions/mean_length": 18.71875, "completions/mean_terminated_length": 17.807693481445312, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9773024804890156, "epoch": 0.00276, "frac_reward_zero_std": 0.0, "grad_norm": 0.2592693567276001, "kl": 0.25720518734306097, "learning_rate": 9.745611942489202e-06, "loss": -0.1276, "num_tokens": 6177147.0, "reward": -0.078125, "reward_std": 0.3971337080001831, "rewards/rollout_reward_func/mean": -0.078125, "rewards/rollout_reward_func/std": 0.3866601288318634, "sampling/importance_sampling_ratio/max": 1.9973775148391724, "sampling/importance_sampling_ratio/mean": 0.386565238237381, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.09134292602539, "sampling/sampling_logp_difference/mean": 2.008255958557129, "step": 138, "step_time": 19.47141169400129 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.005222323001362383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006581018678843975, "completions/clipped_ratio": 0.25, "completions/max_length": 29.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.46875, "completions/mean_terminated_length": 18.75, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.0551363546401262, "epoch": 0.00278, "frac_reward_zero_std": 0.0, "grad_norm": 0.4120376408100128, "kl": 0.624184443615377, "learning_rate": 9.740658072783244e-06, "loss": -0.1453, "num_tokens": 6222043.0, "reward": -0.14687499403953552, "reward_std": 0.2827109694480896, "rewards/rollout_reward_func/mean": -0.14687499403953552, "rewards/rollout_reward_func/std": 0.2839631736278534, "sampling/importance_sampling_ratio/max": 2.2330379486083984, "sampling/importance_sampling_ratio/mean": 0.5451232194900513, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.40774154663086, "sampling/sampling_logp_difference/mean": 2.004826784133911, "step": 139, "step_time": 18.187440927998978 }, { "clip_ratio/high_max": 0.005965909222140908, "clip_ratio/high_mean": 0.002982954611070454, "clip_ratio/low_mean": 0.0030598959419876337, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006042850553058088, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 19.25, "completions/mean_terminated_length": 18.041667938232422, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1073652859777212, "epoch": 0.0028, "frac_reward_zero_std": 0.0, "grad_norm": 0.2620665431022644, "kl": 0.8314186949282885, "learning_rate": 9.73565815893761e-06, "loss": -0.1362, "num_tokens": 6266992.0, "reward": -0.11562499403953552, "reward_std": 0.3190055787563324, "rewards/rollout_reward_func/mean": -0.11562499403953552, "rewards/rollout_reward_func/std": 0.3183722496032715, "sampling/importance_sampling_ratio/max": 2.1219723224639893, "sampling/importance_sampling_ratio/mean": 0.4258824586868286, "sampling/importance_sampling_ratio/min": 2.704506036146897e-43, "sampling/sampling_logp_difference/max": 33.72354507446289, "sampling/sampling_logp_difference/mean": 2.3406589031219482, "step": 140, "step_time": 19.44314586200926 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0006648935959674418, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019148935680277646, "completions/clipped_ratio": 0.125, "completions/max_length": 47.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 18.09375, "completions/mean_terminated_length": 16.25, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8512001466006041, "epoch": 0.00282, "frac_reward_zero_std": 0.0, "grad_norm": 0.27419453859329224, "kl": 0.5638364776968956, "learning_rate": 9.730612266904548e-06, "loss": -0.0916, "num_tokens": 6312352.0, "reward": -0.08750000596046448, "reward_std": 0.29960113763809204, "rewards/rollout_reward_func/mean": -0.08750000596046448, "rewards/rollout_reward_func/std": 0.31597262620925903, "sampling/importance_sampling_ratio/max": 2.122159481048584, "sampling/importance_sampling_ratio/mean": 0.6321670413017273, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.696319580078125, "sampling/sampling_logp_difference/mean": 1.812274694442749, "step": 141, "step_time": 19.638382197008468 }, { "clip_ratio/high_max": 0.006987179629504681, "clip_ratio/high_mean": 0.0034935898147523403, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034935898147523403, "completions/clipped_ratio": 0.09375, "completions/max_length": 30.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 15.1875, "completions/mean_terminated_length": 14.620689392089844, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8643730580806732, "epoch": 0.00284, "frac_reward_zero_std": 0.0, "grad_norm": 0.3194446861743927, "kl": 0.38593083596788347, "learning_rate": 9.72552046324278e-06, "loss": -0.0852, "num_tokens": 6356315.0, "reward": 0.03437500074505806, "reward_std": 0.2516579329967499, "rewards/rollout_reward_func/mean": 0.03437500074505806, "rewards/rollout_reward_func/std": 0.2509457767009735, "sampling/importance_sampling_ratio/max": 1.8614962100982666, "sampling/importance_sampling_ratio/mean": 0.5943940877914429, "sampling/importance_sampling_ratio/min": 2.942726775082116e-44, "sampling/sampling_logp_difference/max": 33.67109680175781, "sampling/sampling_logp_difference/mean": 2.174816131591797, "step": 142, "step_time": 18.26121768995654 }, { "clip_ratio/high_max": 0.005217391299083829, "clip_ratio/high_mean": 0.0038586956216022372, "clip_ratio/low_mean": 0.005361842107959092, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009220537729561329, "completions/clipped_ratio": 0.09375, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 17.9375, "completions/mean_terminated_length": 17.482759475708008, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0763911083340645, "epoch": 0.00286, "frac_reward_zero_std": 0.0, "grad_norm": 0.2850593030452728, "kl": 0.4679091388825327, "learning_rate": 9.72038281511664e-06, "loss": -0.0152, "num_tokens": 6401456.0, "reward": 0.07187500596046448, "reward_std": 0.341460645198822, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.35033106803894043, "sampling/importance_sampling_ratio/max": 1.770760178565979, "sampling/importance_sampling_ratio/mean": 0.44086432456970215, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.52471923828125, "sampling/sampling_logp_difference/mean": 2.4487314224243164, "step": 143, "step_time": 19.156973587960238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.1875, "completions/max_length": 60.0, "completions/max_terminated_length": 60.0, "completions/mean_length": 18.625, "completions/mean_terminated_length": 17.615385055541992, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.8372380416840315, "epoch": 0.00288, "frac_reward_zero_std": 0.0, "grad_norm": 0.33522868156433105, "kl": 0.17102638445794582, "learning_rate": 9.715199390295181e-06, "loss": -0.1576, "num_tokens": 6446356.0, "reward": -0.11249999701976776, "reward_std": 0.3216397762298584, "rewards/rollout_reward_func/mean": -0.11249999701976776, "rewards/rollout_reward_func/std": 0.3367300033569336, "sampling/importance_sampling_ratio/max": 2.0573904514312744, "sampling/importance_sampling_ratio/mean": 0.6782102584838867, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.53605270385742, "sampling/sampling_logp_difference/mean": 1.4870193004608154, "step": 144, "step_time": 19.300980055020773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0055509868543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0055509868543595076, "completions/clipped_ratio": 0.21875, "completions/max_length": 47.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.46875, "completions/mean_terminated_length": 16.68000030517578, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0209394060075283, "epoch": 0.0029, "frac_reward_zero_std": 0.0, "grad_norm": 0.36020323634147644, "kl": 0.42030554288066924, "learning_rate": 9.70997025715128e-06, "loss": -0.1144, "num_tokens": 6492087.0, "reward": -0.03750000149011612, "reward_std": 0.3425980806350708, "rewards/rollout_reward_func/mean": -0.03750000149011612, "rewards/rollout_reward_func/std": 0.3544646203517914, "sampling/importance_sampling_ratio/max": 2.1791508197784424, "sampling/importance_sampling_ratio/mean": 0.49679261445999146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.45139694213867, "sampling/sampling_logp_difference/mean": 1.999082326889038, "step": 145, "step_time": 18.852003615014837 }, { "clip_ratio/high_max": 0.026664976496249437, "clip_ratio/high_mean": 0.014894988271407783, "clip_ratio/low_mean": 0.0022580644581466913, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017153052613139153, "completions/clipped_ratio": 0.25, "completions/max_length": 48.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 21.875, "completions/mean_terminated_length": 20.666667938232422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1369945583865047, "epoch": 0.00292, "frac_reward_zero_std": 0.0, "grad_norm": 0.1826665848493576, "kl": 0.393000305397436, "learning_rate": 9.704695484660736e-06, "loss": -0.0616, "num_tokens": 6537709.0, "reward": -0.05937499925494194, "reward_std": 0.3794671297073364, "rewards/rollout_reward_func/mean": -0.05937499925494194, "rewards/rollout_reward_func/std": 0.3859294354915619, "sampling/importance_sampling_ratio/max": 1.9330321550369263, "sampling/importance_sampling_ratio/mean": 0.24438008666038513, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.80073547363281, "sampling/sampling_logp_difference/mean": 2.239119529724121, "step": 146, "step_time": 19.80259898400982 }, { "clip_ratio/high_max": 0.012885552365332842, "clip_ratio/high_mean": 0.006442776182666421, "clip_ratio/low_mean": 0.002582282992079854, "clip_ratio/low_min": 0.0014880952658131719, "clip_ratio/region_mean": 0.009025059174746275, "completions/clipped_ratio": 0.3125, "completions/max_length": 48.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 18.40625, "completions/mean_terminated_length": 16.772727966308594, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9579877350479364, "epoch": 0.00294, "frac_reward_zero_std": 0.0, "grad_norm": 0.2366267442703247, "kl": 0.4405937073752284, "learning_rate": 9.699375142401365e-06, "loss": -0.203, "num_tokens": 6581111.0, "reward": -0.15312500298023224, "reward_std": 0.3492256999015808, "rewards/rollout_reward_func/mean": -0.15312500298023224, "rewards/rollout_reward_func/std": 0.35919657349586487, "sampling/importance_sampling_ratio/max": 1.9249444007873535, "sampling/importance_sampling_ratio/mean": 0.48179301619529724, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.997589111328125, "sampling/sampling_logp_difference/mean": 1.6967954635620117, "step": 147, "step_time": 19.227099612966413 }, { "clip_ratio/high_max": 0.009599831886589527, "clip_ratio/high_mean": 0.0047999159432947636, "clip_ratio/low_mean": 0.003258689888752997, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00805860583204776, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.3125, "completions/mean_terminated_length": 16.178571701049805, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.948581563308835, "epoch": 0.00296, "frac_reward_zero_std": 0.0, "grad_norm": 0.21033340692520142, "kl": 0.28832459822297096, "learning_rate": 9.694009300552081e-06, "loss": -0.114, "num_tokens": 6626197.0, "reward": -0.0625, "reward_std": 0.3935985565185547, "rewards/rollout_reward_func/mean": -0.0625, "rewards/rollout_reward_func/std": 0.3790905773639679, "sampling/importance_sampling_ratio/max": 1.9247554540634155, "sampling/importance_sampling_ratio/mean": 0.5759155750274658, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.32164001464844, "sampling/sampling_logp_difference/mean": 2.1950879096984863, "step": 148, "step_time": 18.975694955981453 }, { "clip_ratio/high_max": 0.009078947361558676, "clip_ratio/high_mean": 0.004539473680779338, "clip_ratio/low_mean": 0.005264945677481592, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00980441935826093, "completions/clipped_ratio": 0.15625, "completions/max_length": 31.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 16.5, "completions/mean_terminated_length": 15.666666984558105, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9237694535404444, "epoch": 0.00298, "frac_reward_zero_std": 0.0, "grad_norm": 0.23428449034690857, "kl": 0.30079783988185227, "learning_rate": 9.68859802989196e-06, "loss": -0.0923, "num_tokens": 6670865.0, "reward": -0.015625, "reward_std": 0.35337603092193604, "rewards/rollout_reward_func/mean": -0.015625, "rewards/rollout_reward_func/std": 0.3427680432796478, "sampling/importance_sampling_ratio/max": 2.356343984603882, "sampling/importance_sampling_ratio/mean": 0.5801748037338257, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.211952209472656, "sampling/sampling_logp_difference/mean": 1.901810646057129, "step": 149, "step_time": 18.604433607993997 }, { "clip_ratio/high_max": 0.0036764706019312143, "clip_ratio/high_mean": 0.0018382353009656072, "clip_ratio/low_mean": 0.0033356506610289216, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005173885961994529, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 19.03125, "completions/mean_terminated_length": 17.428571701049805, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.09602627530694, "epoch": 0.003, "frac_reward_zero_std": 0.0, "grad_norm": 0.23610647022724152, "kl": 0.2760507899802178, "learning_rate": 9.683141401799326e-06, "loss": -0.1312, "num_tokens": 6716656.0, "reward": -0.05624999850988388, "reward_std": 0.28264087438583374, "rewards/rollout_reward_func/mean": -0.05624999850988388, "rewards/rollout_reward_func/std": 0.3015417456626892, "sampling/importance_sampling_ratio/max": 1.9430104494094849, "sampling/importance_sampling_ratio/mean": 0.4927893877029419, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.82010269165039, "sampling/sampling_logp_difference/mean": 2.1837942600250244, "step": 150, "step_time": 19.151746966002975 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009171195677481592, "completions/clipped_ratio": 0.09375, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 14.125, "completions/mean_terminated_length": 13.379310607910156, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.6650240262970328, "epoch": 0.00302, "frac_reward_zero_std": 0.0, "grad_norm": 0.28438881039619446, "kl": 0.43690616730600595, "learning_rate": 9.677639488250787e-06, "loss": -0.1672, "num_tokens": 6760176.0, "reward": -0.04062499850988388, "reward_std": 0.3095476031303406, "rewards/rollout_reward_func/mean": -0.04062499850988388, "rewards/rollout_reward_func/std": 0.29823067784309387, "sampling/importance_sampling_ratio/max": 2.1911988258361816, "sampling/importance_sampling_ratio/mean": 0.7058163285255432, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.5394287109375, "sampling/sampling_logp_difference/mean": 1.1373989582061768, "step": 151, "step_time": 17.285226437001256 }, { "clip_ratio/high_max": 0.008482143050059676, "clip_ratio/high_mean": 0.004241071525029838, "clip_ratio/low_mean": 0.003810618771240115, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008051690296269953, "completions/clipped_ratio": 0.125, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 17.03125, "completions/mean_terminated_length": 15.464286804199219, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8882812932133675, "epoch": 0.00304, "frac_reward_zero_std": 0.0, "grad_norm": 0.2779054343700409, "kl": 0.5375087386928499, "learning_rate": 9.672092361820306e-06, "loss": -0.0798, "num_tokens": 6804582.0, "reward": 0.10625000298023224, "reward_std": 0.19597986340522766, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.21089288592338562, "sampling/importance_sampling_ratio/max": 2.2148337364196777, "sampling/importance_sampling_ratio/mean": 0.4863426387310028, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.55569076538086, "sampling/sampling_logp_difference/mean": 1.9994540214538574, "step": 152, "step_time": 19.249376887004473 }, { "clip_ratio/high_max": 0.009010416688397527, "clip_ratio/high_mean": 0.004505208344198763, "clip_ratio/low_mean": 0.002366071450524032, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0068712797947227955, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 19.5625, "completions/mean_terminated_length": 17.653846740722656, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.1168677806854248, "epoch": 0.00306, "frac_reward_zero_std": 0.0, "grad_norm": 0.23736093938350677, "kl": 0.26085410825908184, "learning_rate": 9.666500095678226e-06, "loss": -0.0233, "num_tokens": 6850235.0, "reward": 0.05000000074505806, "reward_std": 0.3205743432044983, "rewards/rollout_reward_func/mean": 0.05000000074505806, "rewards/rollout_reward_func/std": 0.31418097019195557, "sampling/importance_sampling_ratio/max": 2.019132137298584, "sampling/importance_sampling_ratio/mean": 0.4843340516090393, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.180702209472656, "sampling/sampling_logp_difference/mean": 2.4042835235595703, "step": 153, "step_time": 18.974949310024385 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.004092262010090053, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005512716597877443, "completions/clipped_ratio": 0.09375, "completions/max_length": 56.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.40625, "completions/mean_terminated_length": 18.10344886779785, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.009780440479517, "epoch": 0.00308, "frac_reward_zero_std": 0.0, "grad_norm": 0.260113388299942, "kl": 0.3184106759727001, "learning_rate": 9.660862763590322e-06, "loss": -0.1223, "num_tokens": 6894989.0, "reward": 0.02812500111758709, "reward_std": 0.3638427257537842, "rewards/rollout_reward_func/mean": 0.02812500111758709, "rewards/rollout_reward_func/std": 0.365649551153183, "sampling/importance_sampling_ratio/max": 1.6576006412506104, "sampling/importance_sampling_ratio/mean": 0.38986721634864807, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.851356506347656, "sampling/sampling_logp_difference/mean": 2.159792900085449, "step": 154, "step_time": 19.69534919898433 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.0026041667442768812, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003806089865975082, "completions/clipped_ratio": 0.15625, "completions/max_length": 34.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 17.78125, "completions/mean_terminated_length": 16.037036895751953, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8948944061994553, "epoch": 0.0031, "frac_reward_zero_std": 0.0, "grad_norm": 0.2801826298236847, "kl": 1.333799850428477, "learning_rate": 9.655180439916814e-06, "loss": -0.1351, "num_tokens": 6939833.0, "reward": -0.02499999664723873, "reward_std": 0.28955817222595215, "rewards/rollout_reward_func/mean": -0.02499999664723873, "rewards/rollout_reward_func/std": 0.3110854923725128, "sampling/importance_sampling_ratio/max": 2.017792224884033, "sampling/importance_sampling_ratio/mean": 0.44747331738471985, "sampling/importance_sampling_ratio/min": 1.5274153261140506e-43, "sampling/sampling_logp_difference/max": 32.993186950683594, "sampling/sampling_logp_difference/mean": 1.7615423202514648, "step": 155, "step_time": 19.267849448980996 }, { "clip_ratio/high_max": 0.005217391299083829, "clip_ratio/high_mean": 0.0026086956495419145, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026086956495419145, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 18.25, "completions/mean_terminated_length": 15.769231796264648, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.9700934831053019, "epoch": 0.00312, "frac_reward_zero_std": 0.0, "grad_norm": 0.22078944742679596, "kl": 0.23766211094334722, "learning_rate": 9.64945319961139e-06, "loss": -0.0576, "num_tokens": 6985842.0, "reward": -0.046875, "reward_std": 0.3340074419975281, "rewards/rollout_reward_func/mean": -0.046875, "rewards/rollout_reward_func/std": 0.34076231718063354, "sampling/importance_sampling_ratio/max": 2.225569725036621, "sampling/importance_sampling_ratio/mean": 0.5468417406082153, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.344276428222656, "sampling/sampling_logp_difference/mean": 2.101065158843994, "step": 156, "step_time": 18.89411428103631 }, { "clip_ratio/high_max": 0.0022321429569274187, "clip_ratio/high_mean": 0.0011160714784637094, "clip_ratio/low_mean": 0.0024519230937585235, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003567994572222233, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 16.46875, "completions/mean_terminated_length": 14.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8659570384770632, "epoch": 0.00314, "frac_reward_zero_std": 0.0, "grad_norm": 0.29986023902893066, "kl": 0.34679764322936535, "learning_rate": 9.643681118220224e-06, "loss": -0.1424, "num_tokens": 7029232.0, "reward": 0.02187500149011612, "reward_std": 0.36691635847091675, "rewards/rollout_reward_func/mean": 0.02187500149011612, "rewards/rollout_reward_func/std": 0.3607647716999054, "sampling/importance_sampling_ratio/max": 1.8513425588607788, "sampling/importance_sampling_ratio/mean": 0.6218792200088501, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 33.17667770385742, "sampling/sampling_logp_difference/mean": 1.49717378616333, "step": 157, "step_time": 18.859768928959966 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.003910779021680355, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005112702143378556, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 15.96875, "completions/mean_terminated_length": 14.88888931274414, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7351051410660148, "epoch": 0.00316, "frac_reward_zero_std": 0.0, "grad_norm": 0.2141881287097931, "kl": 0.44942968152463436, "learning_rate": 9.637864271880972e-06, "loss": -0.1947, "num_tokens": 7071875.0, "reward": 0.0312500074505806, "reward_std": 0.31385916471481323, "rewards/rollout_reward_func/mean": 0.0312500074505806, "rewards/rollout_reward_func/std": 0.329650342464447, "sampling/importance_sampling_ratio/max": 1.9187161922454834, "sampling/importance_sampling_ratio/mean": 0.5777536630630493, "sampling/importance_sampling_ratio/min": 1.2611686178923354e-44, "sampling/sampling_logp_difference/max": 33.0605583190918, "sampling/sampling_logp_difference/mean": 1.4736628532409668, "step": 158, "step_time": 18.306852702997276 }, { "clip_ratio/high_max": 0.0077838830184191465, "clip_ratio/high_mean": 0.0038919415092095733, "clip_ratio/low_mean": 0.0018841515993699431, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005776093108579516, "completions/clipped_ratio": 0.15625, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 17.34375, "completions/mean_terminated_length": 15.037036895751953, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.898386082611978, "epoch": 0.00318, "frac_reward_zero_std": 0.0, "grad_norm": 0.19954800605773926, "kl": 0.2989301490597427, "learning_rate": 9.632002737321768e-06, "loss": -0.2024, "num_tokens": 7116274.0, "reward": -0.04374999552965164, "reward_std": 0.34948667883872986, "rewards/rollout_reward_func/mean": -0.04374999552965164, "rewards/rollout_reward_func/std": 0.37238335609436035, "sampling/importance_sampling_ratio/max": 1.9305213689804077, "sampling/importance_sampling_ratio/mean": 0.5361403822898865, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.379798889160156, "sampling/sampling_logp_difference/mean": 1.9561243057250977, "step": 159, "step_time": 18.625602475964115 }, { "clip_ratio/high_max": 0.013928571715950966, "clip_ratio/high_mean": 0.006964285857975483, "clip_ratio/low_mean": 0.0026086956495419145, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009572981507517397, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 16.0, "completions/mean_terminated_length": 15.448275566101074, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.8546733632683754, "epoch": 0.0032, "frac_reward_zero_std": 0.0, "grad_norm": 0.2933034300804138, "kl": 0.44303234945982695, "learning_rate": 9.626096591860215e-06, "loss": -0.1261, "num_tokens": 7160499.0, "reward": -0.009375005960464478, "reward_std": 0.29474493861198425, "rewards/rollout_reward_func/mean": -0.009375005960464478, "rewards/rollout_reward_func/std": 0.32264968752861023, "sampling/importance_sampling_ratio/max": 1.8984472751617432, "sampling/importance_sampling_ratio/mean": 0.524329423904419, "sampling/importance_sampling_ratio/min": 3.4738188930612215e-42, "sampling/sampling_logp_difference/max": 33.41104507446289, "sampling/sampling_logp_difference/mean": 1.6763722896575928, "step": 160, "step_time": 18.55905507497664 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.010064102709293365, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011552197975106537, "completions/clipped_ratio": 0.09375, "completions/max_length": 27.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 16.03125, "completions/mean_terminated_length": 15.034482955932617, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.03687874507159, "epoch": 0.00322, "frac_reward_zero_std": 0.0, "grad_norm": 0.19759218394756317, "kl": 0.4011722686700523, "learning_rate": 9.620145913402366e-06, "loss": -0.1202, "num_tokens": 7205021.0, "reward": -0.00937500037252903, "reward_std": 0.3511350154876709, "rewards/rollout_reward_func/mean": -0.00937500037252903, "rewards/rollout_reward_func/std": 0.34111711382865906, "sampling/importance_sampling_ratio/max": 2.0392773151397705, "sampling/importance_sampling_ratio/mean": 0.4799976348876953, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.74806594848633, "sampling/sampling_logp_difference/mean": 2.646238327026367, "step": 161, "step_time": 17.66510024499439 }, { "clip_ratio/high_max": 0.004814814776182175, "clip_ratio/high_mean": 0.0024074073880910873, "clip_ratio/low_mean": 0.0011160714784637094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035234788665547967, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.5, "completions/mean_terminated_length": 16.230770111083984, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9930814206600189, "epoch": 0.00324, "frac_reward_zero_std": 0.0, "grad_norm": 0.27197471261024475, "kl": 0.3530912813730538, "learning_rate": 9.614150780441687e-06, "loss": -0.1246, "num_tokens": 7249278.0, "reward": 0.003125002607703209, "reward_std": 0.28250420093536377, "rewards/rollout_reward_func/mean": 0.003125002607703209, "rewards/rollout_reward_func/std": 0.28225404024124146, "sampling/importance_sampling_ratio/max": 2.0073344707489014, "sampling/importance_sampling_ratio/mean": 0.5259261131286621, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.94546127319336, "sampling/sampling_logp_difference/mean": 2.1113624572753906, "step": 162, "step_time": 18.47736286003783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0038586956216022372, "clip_ratio/low_min": 0.0024999999441206455, "clip_ratio/region_mean": 0.0038586956216022372, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 17.90625, "completions/mean_terminated_length": 15.869565963745117, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.8480937872081995, "epoch": 0.00326, "frac_reward_zero_std": 0.0, "grad_norm": 0.33003875613212585, "kl": 0.3055895734578371, "learning_rate": 9.608111272058044e-06, "loss": -0.1672, "num_tokens": 7293839.0, "reward": -0.13750000298023224, "reward_std": 0.34798264503479004, "rewards/rollout_reward_func/mean": -0.13750000298023224, "rewards/rollout_reward_func/std": 0.3414863348007202, "sampling/importance_sampling_ratio/max": 2.0785655975341797, "sampling/importance_sampling_ratio/mean": 0.6528178453445435, "sampling/importance_sampling_ratio/min": 1.1543896749107843e-41, "sampling/sampling_logp_difference/max": 32.33570098876953, "sampling/sampling_logp_difference/mean": 1.4635976552963257, "step": 163, "step_time": 18.665469261992257 }, { "clip_ratio/high_max": 0.009548611240461469, "clip_ratio/high_mean": 0.004774305620230734, "clip_ratio/low_mean": 0.0033735795877873898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008147885208018124, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.40625, "completions/mean_terminated_length": 16.520000457763672, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0547459311783314, "epoch": 0.00328, "frac_reward_zero_std": 0.0, "grad_norm": 0.14810746908187866, "kl": 0.30075165582820773, "learning_rate": 9.602027467916626e-06, "loss": -0.0728, "num_tokens": 7339141.0, "reward": 0.0062500061467289925, "reward_std": 0.319302499294281, "rewards/rollout_reward_func/mean": 0.0062500061467289925, "rewards/rollout_reward_func/std": 0.3291607201099396, "sampling/importance_sampling_ratio/max": 2.176086187362671, "sampling/importance_sampling_ratio/mean": 0.374366819858551, "sampling/importance_sampling_ratio/min": 2.1551970381315687e-42, "sampling/sampling_logp_difference/max": 32.85859680175781, "sampling/sampling_logp_difference/mean": 1.9583611488342285, "step": 164, "step_time": 18.757072626045556 }, { "clip_ratio/high_max": 0.006801470648497343, "clip_ratio/high_mean": 0.0034007353242486715, "clip_ratio/low_mean": 0.0034722222480922937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006872957572340965, "completions/clipped_ratio": 0.125, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 14.34375, "completions/mean_terminated_length": 14.10714340209961, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.8139143297448754, "epoch": 0.0033, "frac_reward_zero_std": 0.0, "grad_norm": 0.37560081481933594, "kl": 0.7561982357874513, "learning_rate": 9.595899448266928e-06, "loss": -0.0993, "num_tokens": 7383064.0, "reward": -0.078125, "reward_std": 0.3087320923805237, "rewards/rollout_reward_func/mean": -0.078125, "rewards/rollout_reward_func/std": 0.30132436752319336, "sampling/importance_sampling_ratio/max": 2.8073463439941406, "sampling/importance_sampling_ratio/mean": 0.7344714403152466, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.53569412231445, "sampling/sampling_logp_difference/mean": 1.8475403785705566, "step": 165, "step_time": 18.718999062970397 }, { "clip_ratio/high_max": 0.0061011905781924725, "clip_ratio/high_mean": 0.0030505952890962362, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004208002705127001, "completions/clipped_ratio": 0.15625, "completions/max_length": 41.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.40625, "completions/mean_terminated_length": 16.148147583007812, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.9294176641851664, "epoch": 0.00332, "frac_reward_zero_std": 0.0, "grad_norm": 0.215289905667305, "kl": 0.3191096498630941, "learning_rate": 9.589727293941669e-06, "loss": -0.1345, "num_tokens": 7427539.0, "reward": -0.031249992549419403, "reward_std": 0.30705463886260986, "rewards/rollout_reward_func/mean": -0.031249992549419403, "rewards/rollout_reward_func/std": 0.3325730562210083, "sampling/importance_sampling_ratio/max": 1.8682342767715454, "sampling/importance_sampling_ratio/mean": 0.5983273983001709, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.65547561645508, "sampling/sampling_logp_difference/mean": 1.9331669807434082, "step": 166, "step_time": 19.04854785600037 }, { "clip_ratio/high_max": 0.016549809370189905, "clip_ratio/high_mean": 0.008274904685094953, "clip_ratio/low_mean": 0.008969907416030765, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017244811984710395, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 20.96875, "completions/mean_terminated_length": 19.239999771118164, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2594295926392078, "epoch": 0.00334, "frac_reward_zero_std": 0.0, "grad_norm": 0.2629305422306061, "kl": 0.347363939974457, "learning_rate": 9.583511086355738e-06, "loss": -0.1261, "num_tokens": 7473856.0, "reward": -0.0625, "reward_std": 0.40887975692749023, "rewards/rollout_reward_func/mean": -0.0625, "rewards/rollout_reward_func/std": 0.3957434892654419, "sampling/importance_sampling_ratio/max": 1.8965973854064941, "sampling/importance_sampling_ratio/mean": 0.3571093678474426, "sampling/importance_sampling_ratio/min": 3.2440059449119515e-42, "sampling/sampling_logp_difference/max": 33.3054084777832, "sampling/sampling_logp_difference/mean": 2.5305933952331543, "step": 167, "step_time": 19.707448145971284 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0034420291194692254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00459943653549999, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.84375, "completions/mean_terminated_length": 17.76923179626465, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.125005355104804, "epoch": 0.00336, "frac_reward_zero_std": 0.0, "grad_norm": 0.351022332906723, "kl": 0.2957208906300366, "learning_rate": 9.577250907505117e-06, "loss": -0.092, "num_tokens": 7519161.0, "reward": -0.09062500298023224, "reward_std": 0.30344605445861816, "rewards/rollout_reward_func/mean": -0.09062500298023224, "rewards/rollout_reward_func/std": 0.3196362555027008, "sampling/importance_sampling_ratio/max": 1.6685582399368286, "sampling/importance_sampling_ratio/mean": 0.36878857016563416, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.355918884277344, "sampling/sampling_logp_difference/mean": 2.3346023559570312, "step": 168, "step_time": 18.8907985509868 }, { "clip_ratio/high_max": 0.006868132157251239, "clip_ratio/high_mean": 0.0034340660786256194, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004591473494656384, "completions/clipped_ratio": 0.125, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 16.8125, "completions/mean_terminated_length": 15.85714340209961, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7681588269770145, "epoch": 0.00338, "frac_reward_zero_std": 0.0, "grad_norm": 0.18586498498916626, "kl": 0.3012585889082402, "learning_rate": 9.570946839965795e-06, "loss": -0.1513, "num_tokens": 7562951.0, "reward": 0.01562500186264515, "reward_std": 0.36395972967147827, "rewards/rollout_reward_func/mean": 0.01562500186264515, "rewards/rollout_reward_func/std": 0.3556995391845703, "sampling/importance_sampling_ratio/max": 1.9299520254135132, "sampling/importance_sampling_ratio/mean": 0.5162689685821533, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.273799896240234, "sampling/sampling_logp_difference/mean": 1.9404332637786865, "step": 169, "step_time": 18.400136743002804 }, { "clip_ratio/high_max": 0.009956140536814928, "clip_ratio/high_mean": 0.004978070268407464, "clip_ratio/low_mean": 0.0036766902776435018, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008654760546050966, "completions/clipped_ratio": 0.21875, "completions/max_length": 56.0, "completions/max_terminated_length": 56.0, "completions/mean_length": 19.03125, "completions/mean_terminated_length": 17.84000015258789, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0267216227948666, "epoch": 0.0034, "frac_reward_zero_std": 0.0, "grad_norm": 0.19181574881076813, "kl": 0.5756668150424957, "learning_rate": 9.564598966892683e-06, "loss": -0.2188, "num_tokens": 7608625.0, "reward": -0.15312500298023224, "reward_std": 0.3557414710521698, "rewards/rollout_reward_func/mean": -0.15312500298023224, "rewards/rollout_reward_func/std": 0.36188071966171265, "sampling/importance_sampling_ratio/max": 1.83348548412323, "sampling/importance_sampling_ratio/mean": 0.5152292251586914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.55074691772461, "sampling/sampling_logp_difference/mean": 2.2056963443756104, "step": 170, "step_time": 19.500052128976677 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.0036122312303632498, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00468981743324548, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.03125, "completions/mean_terminated_length": 15.629630088806152, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1148479953408241, "epoch": 0.00342, "frac_reward_zero_std": 0.0, "grad_norm": 0.13112381100654602, "kl": 0.4538731970824301, "learning_rate": 9.558207372018519e-06, "loss": -0.053, "num_tokens": 7654400.0, "reward": -0.05312500149011612, "reward_std": 0.3070458769798279, "rewards/rollout_reward_func/mean": -0.05312500149011612, "rewards/rollout_reward_func/std": 0.3182455897331238, "sampling/importance_sampling_ratio/max": 2.507512092590332, "sampling/importance_sampling_ratio/mean": 0.5256720781326294, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.434043884277344, "sampling/sampling_logp_difference/mean": 2.410623073577881, "step": 171, "step_time": 18.75984157102357 }, { "clip_ratio/high_max": 0.01785714365541935, "clip_ratio/high_mean": 0.008928571827709675, "clip_ratio/low_mean": 0.0007621950935572386, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009690766921266913, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 16.90625, "completions/mean_terminated_length": 15.076923370361328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8841255493462086, "epoch": 0.00344, "frac_reward_zero_std": 0.0, "grad_norm": 0.2197365015745163, "kl": 0.688818760914728, "learning_rate": 9.551772139652762e-06, "loss": -0.0934, "num_tokens": 7698810.0, "reward": -0.03437500074505806, "reward_std": 0.3777256906032562, "rewards/rollout_reward_func/mean": -0.03437500074505806, "rewards/rollout_reward_func/std": 0.3703610897064209, "sampling/importance_sampling_ratio/max": 2.5138211250305176, "sampling/importance_sampling_ratio/mean": 0.4744627773761749, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.3107795715332, "sampling/sampling_logp_difference/mean": 1.7932928800582886, "step": 172, "step_time": 18.879767547012307 }, { "clip_ratio/high_max": 0.005624999990686774, "clip_ratio/high_mean": 0.002812499995343387, "clip_ratio/low_mean": 0.0020833334419876337, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004895833437331021, "completions/clipped_ratio": 0.125, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 17.71875, "completions/mean_terminated_length": 16.35714340209961, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9223120287060738, "epoch": 0.00346, "frac_reward_zero_std": 0.0, "grad_norm": 0.37365779280662537, "kl": 0.4283573403954506, "learning_rate": 9.545293354680477e-06, "loss": -0.042, "num_tokens": 7744455.0, "reward": -0.125, "reward_std": 0.3416116535663605, "rewards/rollout_reward_func/mean": -0.125, "rewards/rollout_reward_func/std": 0.33696940541267395, "sampling/importance_sampling_ratio/max": 2.5971107482910156, "sampling/importance_sampling_ratio/mean": 0.4204016625881195, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.586307525634766, "sampling/sampling_logp_difference/mean": 1.8959462642669678, "step": 173, "step_time": 18.69946070201695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 15.84375, "completions/mean_terminated_length": 14.269230842590332, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.5893385671079159, "epoch": 0.00348, "frac_reward_zero_std": 0.0, "grad_norm": 0.40366601943969727, "kl": 0.39308660198003054, "learning_rate": 9.538771102561212e-06, "loss": -0.0661, "num_tokens": 7787992.0, "reward": 0.009375002235174179, "reward_std": 0.2946464419364929, "rewards/rollout_reward_func/mean": 0.009375002235174179, "rewards/rollout_reward_func/std": 0.30623558163642883, "sampling/importance_sampling_ratio/max": 1.8369942903518677, "sampling/importance_sampling_ratio/mean": 0.7637128829956055, "sampling/importance_sampling_ratio/min": 5.072700440855838e-43, "sampling/sampling_logp_difference/max": 31.976245880126953, "sampling/sampling_logp_difference/mean": 1.0754549503326416, "step": 174, "step_time": 18.65194725697802 }, { "clip_ratio/high_max": 0.01092748437076807, "clip_ratio/high_mean": 0.0065413282718509436, "clip_ratio/low_mean": 0.0035714287078008056, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01011275697965175, "completions/clipped_ratio": 0.125, "completions/max_length": 47.0, "completions/max_terminated_length": 47.0, "completions/mean_length": 17.59375, "completions/mean_terminated_length": 15.642857551574707, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.944964936003089, "epoch": 0.0035, "frac_reward_zero_std": 0.0, "grad_norm": 0.21932663023471832, "kl": 0.3074033120647073, "learning_rate": 9.53220546932789e-06, "loss": -0.1961, "num_tokens": 7832066.0, "reward": -0.0625, "reward_std": 0.32394784688949585, "rewards/rollout_reward_func/mean": -0.0625, "rewards/rollout_reward_func/std": 0.316991925239563, "sampling/importance_sampling_ratio/max": 2.035560369491577, "sampling/importance_sampling_ratio/mean": 0.4905432462692261, "sampling/importance_sampling_ratio/min": 2.0833104269117055e-41, "sampling/sampling_logp_difference/max": 33.418418884277344, "sampling/sampling_logp_difference/mean": 1.9301706552505493, "step": 175, "step_time": 18.807547385062207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0022916666930541396, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022916666930541396, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.21875, "completions/mean_terminated_length": 15.799999237060547, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8855715394020081, "epoch": 0.00352, "frac_reward_zero_std": 0.0, "grad_norm": 0.21544671058654785, "kl": 0.29835548251867294, "learning_rate": 9.525596541585646e-06, "loss": -0.0675, "num_tokens": 7877019.0, "reward": 0.006250001490116119, "reward_std": 0.30379319190979004, "rewards/rollout_reward_func/mean": 0.006250001490116119, "rewards/rollout_reward_func/std": 0.31718266010284424, "sampling/importance_sampling_ratio/max": 2.8799686431884766, "sampling/importance_sampling_ratio/mean": 0.5241162776947021, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.48091506958008, "sampling/sampling_logp_difference/mean": 1.7244517803192139, "step": 176, "step_time": 18.917196617039735 }, { "clip_ratio/high_max": 0.0044531249441206455, "clip_ratio/high_mean": 0.0022265624720603228, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022265624720603228, "completions/clipped_ratio": 0.0625, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 17.28125, "completions/mean_terminated_length": 16.83333396911621, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8995252009481192, "epoch": 0.00354, "frac_reward_zero_std": 0.0, "grad_norm": 0.21135035157203674, "kl": 0.3013985666912049, "learning_rate": 9.518944406510707e-06, "loss": -0.1405, "num_tokens": 7922024.0, "reward": -0.01874999888241291, "reward_std": 0.3193172812461853, "rewards/rollout_reward_func/mean": -0.01874999888241291, "rewards/rollout_reward_func/std": 0.32768741250038147, "sampling/importance_sampling_ratio/max": 2.0337464809417725, "sampling/importance_sampling_ratio/mean": 0.5837673544883728, "sampling/importance_sampling_ratio/min": 3.1669345293740866e-43, "sampling/sampling_logp_difference/max": 33.04608917236328, "sampling/sampling_logp_difference/mean": 1.932084083557129, "step": 177, "step_time": 19.21186505202786 }, { "clip_ratio/high_max": 0.016361702466383576, "clip_ratio/high_mean": 0.008180851233191788, "clip_ratio/low_mean": 0.0039291438879445195, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012109995179343969, "completions/clipped_ratio": 0.21875, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 19.25, "completions/mean_terminated_length": 18.31999969482422, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.2689857743680477, "epoch": 0.00356, "frac_reward_zero_std": 0.0, "grad_norm": 0.265472948551178, "kl": 0.3040296589024365, "learning_rate": 9.512249151849229e-06, "loss": -0.0504, "num_tokens": 7967434.0, "reward": -0.04999999329447746, "reward_std": 0.4006991982460022, "rewards/rollout_reward_func/mean": -0.04999999329447746, "rewards/rollout_reward_func/std": 0.3951316773891449, "sampling/importance_sampling_ratio/max": 1.831149697303772, "sampling/importance_sampling_ratio/mean": 0.3833501935005188, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.49922561645508, "sampling/sampling_logp_difference/mean": 2.6133530139923096, "step": 178, "step_time": 19.008374553974136 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.003967391327023506, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007873641327023506, "completions/clipped_ratio": 0.09375, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 18.0625, "completions/mean_terminated_length": 18.034482955932617, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9542958457022905, "epoch": 0.00358, "frac_reward_zero_std": 0.0, "grad_norm": 0.2114364206790924, "kl": 0.26243060221895576, "learning_rate": 9.50551086591615e-06, "loss": -0.0836, "num_tokens": 8011951.0, "reward": -0.012500002980232239, "reward_std": 0.2690507471561432, "rewards/rollout_reward_func/mean": -0.012500002980232239, "rewards/rollout_reward_func/std": 0.27911147475242615, "sampling/importance_sampling_ratio/max": 2.324016571044922, "sampling/importance_sampling_ratio/mean": 0.6029808521270752, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.899776458740234, "sampling/sampling_logp_difference/mean": 2.1427059173583984, "step": 179, "step_time": 19.06148032702913 }, { "clip_ratio/high_max": 0.010349026415497065, "clip_ratio/high_mean": 0.005174513207748532, "clip_ratio/low_mean": 0.0043903186451643705, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009564831852912903, "completions/clipped_ratio": 0.28125, "completions/max_length": 47.0, "completions/max_terminated_length": 47.0, "completions/mean_length": 20.25, "completions/mean_terminated_length": 18.69565200805664, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1123284110799432, "epoch": 0.0036, "frac_reward_zero_std": 0.0, "grad_norm": 0.384306937456131, "kl": 0.23193183867260814, "learning_rate": 9.498729637594016e-06, "loss": -0.1286, "num_tokens": 8057559.0, "reward": -0.08749999850988388, "reward_std": 0.24147796630859375, "rewards/rollout_reward_func/mean": -0.08749999850988388, "rewards/rollout_reward_func/std": 0.27327170968055725, "sampling/importance_sampling_ratio/max": 1.9907665252685547, "sampling/importance_sampling_ratio/mean": 0.4650733470916748, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.621543884277344, "sampling/sampling_logp_difference/mean": 2.3492307662963867, "step": 180, "step_time": 19.24080576999404 }, { "clip_ratio/high_max": 0.005217391299083829, "clip_ratio/high_mean": 0.0026086956495419145, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026086956495419145, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 18.3125, "completions/mean_terminated_length": 17.03999900817871, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1437810771167278, "epoch": 0.00362, "frac_reward_zero_std": 0.0, "grad_norm": 0.1448211818933487, "kl": 0.29121233080513775, "learning_rate": 9.491905556331809e-06, "loss": -0.0384, "num_tokens": 8102948.0, "reward": -0.02812500298023224, "reward_std": 0.3650776445865631, "rewards/rollout_reward_func/mean": -0.02812500298023224, "rewards/rollout_reward_func/std": 0.35125064849853516, "sampling/importance_sampling_ratio/max": 1.6829224824905396, "sampling/importance_sampling_ratio/mean": 0.2680819630622864, "sampling/importance_sampling_ratio/min": 2.8446358825793787e-43, "sampling/sampling_logp_difference/max": 32.79608154296875, "sampling/sampling_logp_difference/mean": 2.005095958709717, "step": 181, "step_time": 19.728512044021045 }, { "clip_ratio/high_max": 0.015913903014734387, "clip_ratio/high_mean": 0.007956951507367194, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007956951507367194, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.28125, "completions/mean_terminated_length": 15.125, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0146040339022875, "epoch": 0.00364, "frac_reward_zero_std": 0.0, "grad_norm": 0.3684225082397461, "kl": 0.5324134584516287, "learning_rate": 9.485038712143778e-06, "loss": -0.0963, "num_tokens": 8148752.0, "reward": -0.05000000447034836, "reward_std": 0.32085129618644714, "rewards/rollout_reward_func/mean": -0.05000000447034836, "rewards/rollout_reward_func/std": 0.31826138496398926, "sampling/importance_sampling_ratio/max": 1.9502410888671875, "sampling/importance_sampling_ratio/mean": 0.3915258049964905, "sampling/importance_sampling_ratio/min": 3.923635700109488e-44, "sampling/sampling_logp_difference/max": 33.16841506958008, "sampling/sampling_logp_difference/mean": 1.8998970985412598, "step": 182, "step_time": 19.086217444040813 }, { "clip_ratio/high_max": 0.010320723755285144, "clip_ratio/high_mean": 0.005160361877642572, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005160361877642572, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.0625, "completions/mean_terminated_length": 17.392858505249023, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.107524573802948, "epoch": 0.00366, "frac_reward_zero_std": 0.0, "grad_norm": 0.27532461285591125, "kl": 0.2790465678554028, "learning_rate": 9.478129195608238e-06, "loss": -0.0496, "num_tokens": 8194904.0, "reward": -0.04062499850988388, "reward_std": 0.34125378727912903, "rewards/rollout_reward_func/mean": -0.04062499850988388, "rewards/rollout_reward_func/std": 0.33394935727119446, "sampling/importance_sampling_ratio/max": 1.7423077821731567, "sampling/importance_sampling_ratio/mean": 0.3546157479286194, "sampling/importance_sampling_ratio/min": 4.8765186558503634e-43, "sampling/sampling_logp_difference/max": 33.65674591064453, "sampling/sampling_logp_difference/mean": 2.0752580165863037, "step": 183, "step_time": 18.384253457013983 }, { "clip_ratio/high_max": 0.004814814776182175, "clip_ratio/high_mean": 0.0024074073880910873, "clip_ratio/low_mean": 0.0018676347099244595, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004275042098015547, "completions/clipped_ratio": 0.21875, "completions/max_length": 64.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 20.21875, "completions/mean_terminated_length": 16.920000076293945, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.974729074165225, "epoch": 0.00368, "frac_reward_zero_std": 0.0, "grad_norm": 0.19774256646633148, "kl": 0.3527274227235466, "learning_rate": 9.471177097866384e-06, "loss": -0.1297, "num_tokens": 8239471.0, "reward": -0.11250000447034836, "reward_std": 0.33806461095809937, "rewards/rollout_reward_func/mean": -0.11250000447034836, "rewards/rollout_reward_func/std": 0.3376866281032562, "sampling/importance_sampling_ratio/max": 1.583842158317566, "sampling/importance_sampling_ratio/mean": 0.4327479600906372, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.15430450439453, "sampling/sampling_logp_difference/mean": 1.711584210395813, "step": 184, "step_time": 19.788662875027512 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0024999999441206455, "clip_ratio/low_mean": 0.0009765625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034765624441206455, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 14.1875, "completions/mean_terminated_length": 13.034482955932617, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8781042667105794, "epoch": 0.0037, "frac_reward_zero_std": 0.0, "grad_norm": 0.26023057103157043, "kl": 0.248300991486758, "learning_rate": 9.464182510621085e-06, "loss": -0.1648, "num_tokens": 8283252.0, "reward": 0.02187499776482582, "reward_std": 0.27369481325149536, "rewards/rollout_reward_func/mean": 0.02187499776482582, "rewards/rollout_reward_func/std": 0.30557653307914734, "sampling/importance_sampling_ratio/max": 2.1483757495880127, "sampling/importance_sampling_ratio/mean": 0.6010483503341675, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.80581283569336, "sampling/sampling_logp_difference/mean": 1.943509817123413, "step": 185, "step_time": 18.957623830006924 }, { "clip_ratio/high_max": 0.011594638577662408, "clip_ratio/high_mean": 0.005797319288831204, "clip_ratio/low_mean": 0.0138503402704373, "clip_ratio/low_min": 0.0027173913549631834, "clip_ratio/region_mean": 0.019647659559268504, "completions/clipped_ratio": 0.125, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 18.1875, "completions/mean_terminated_length": 16.892858505249023, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2262501493096352, "epoch": 0.00372, "frac_reward_zero_std": 0.0, "grad_norm": 0.13721366226673126, "kl": 0.26321460073813796, "learning_rate": 9.457145526135673e-06, "loss": -0.055, "num_tokens": 8329239.0, "reward": 0.0031250007450580597, "reward_std": 0.3226400911808014, "rewards/rollout_reward_func/mean": 0.0031250007450580597, "rewards/rollout_reward_func/std": 0.3383874297142029, "sampling/importance_sampling_ratio/max": 1.5601617097854614, "sampling/importance_sampling_ratio/mean": 0.28736889362335205, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.2323112487793, "sampling/sampling_logp_difference/mean": 2.80079984664917, "step": 186, "step_time": 19.445028941016062 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012499999720603228, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 25.0, "completions/mean_length": 13.84375, "completions/mean_terminated_length": 11.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.6708495700731874, "epoch": 0.00374, "frac_reward_zero_std": 0.0, "grad_norm": 0.2903304696083069, "kl": 0.305991995614022, "learning_rate": 9.450066237232736e-06, "loss": -0.1302, "num_tokens": 8371170.0, "reward": 0.028125004842877388, "reward_std": 0.30661630630493164, "rewards/rollout_reward_func/mean": 0.028125004842877388, "rewards/rollout_reward_func/std": 0.30820432305336, "sampling/importance_sampling_ratio/max": 1.8033027648925781, "sampling/importance_sampling_ratio/mean": 0.6693955659866333, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.64875793457031, "sampling/sampling_logp_difference/mean": 1.0646185874938965, "step": 187, "step_time": 18.21215230296366 }, { "clip_ratio/high_max": 0.007708333432674408, "clip_ratio/high_mean": 0.003854166716337204, "clip_ratio/low_mean": 0.008003454538993537, "clip_ratio/low_min": 0.0026041667442768812, "clip_ratio/region_mean": 0.011857621255330741, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 16.65625, "completions/mean_terminated_length": 14.851852416992188, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.941303632222116, "epoch": 0.00376, "frac_reward_zero_std": 0.0, "grad_norm": 0.279154896736145, "kl": 0.37622109381482005, "learning_rate": 9.442944737292872e-06, "loss": -0.0246, "num_tokens": 8415101.0, "reward": 0.09375, "reward_std": 0.261489599943161, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.2861733138561249, "sampling/importance_sampling_ratio/max": 2.109121799468994, "sampling/importance_sampling_ratio/mean": 0.5958802700042725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.117557525634766, "sampling/sampling_logp_difference/mean": 1.9637943506240845, "step": 188, "step_time": 18.29597539799579 }, { "clip_ratio/high_max": 0.014364984119310975, "clip_ratio/high_mean": 0.0071824920596554875, "clip_ratio/low_mean": 0.0026086956495419145, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009791187709197402, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.28125, "completions/mean_terminated_length": 18.079999923706055, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0912352669984102, "epoch": 0.00378, "frac_reward_zero_std": 0.0, "grad_norm": 0.10846090316772461, "kl": 0.24313969374634326, "learning_rate": 9.435781120253486e-06, "loss": -0.0661, "num_tokens": 8459801.0, "reward": -0.03750000149011612, "reward_std": 0.3792235255241394, "rewards/rollout_reward_func/mean": -0.03750000149011612, "rewards/rollout_reward_func/std": 0.363451212644577, "sampling/importance_sampling_ratio/max": 1.6466292142868042, "sampling/importance_sampling_ratio/mean": 0.2791672945022583, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.686729431152344, "sampling/sampling_logp_difference/mean": 2.1631765365600586, "step": 189, "step_time": 19.179177093014005 }, { "clip_ratio/high_max": 0.016856971895322204, "clip_ratio/high_mean": 0.008428485947661102, "clip_ratio/low_mean": 0.0007267441833391786, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00915523013100028, "completions/clipped_ratio": 0.1875, "completions/max_length": 53.0, "completions/max_terminated_length": 53.0, "completions/mean_length": 18.8125, "completions/mean_terminated_length": 16.461538314819336, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9846144802868366, "epoch": 0.0038, "frac_reward_zero_std": 0.0, "grad_norm": 0.36485064029693604, "kl": 0.3670208512339741, "learning_rate": 9.428575480607526e-06, "loss": -0.1025, "num_tokens": 8504188.0, "reward": -0.0468749962747097, "reward_std": 0.36390453577041626, "rewards/rollout_reward_func/mean": -0.0468749962747097, "rewards/rollout_reward_func/std": 0.34825313091278076, "sampling/importance_sampling_ratio/max": 2.4355671405792236, "sampling/importance_sampling_ratio/mean": 0.658850908279419, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.6966552734375, "sampling/sampling_logp_difference/mean": 2.4658150672912598, "step": 190, "step_time": 19.216517088992987 }, { "clip_ratio/high_max": 0.010538949398323894, "clip_ratio/high_mean": 0.005269474699161947, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005269474699161947, "completions/clipped_ratio": 0.1875, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 17.8125, "completions/mean_terminated_length": 16.423076629638672, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9145281445235014, "epoch": 0.00382, "frac_reward_zero_std": 0.0, "grad_norm": 0.4720126986503601, "kl": 0.3407807103358209, "learning_rate": 9.421327913402252e-06, "loss": -0.1339, "num_tokens": 8548551.0, "reward": 0.0, "reward_std": 0.28902146220207214, "rewards/rollout_reward_func/mean": 0.0, "rewards/rollout_reward_func/std": 0.29402655363082886, "sampling/importance_sampling_ratio/max": 2.666325569152832, "sampling/importance_sampling_ratio/mean": 0.4686258137226105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.9935302734375, "sampling/sampling_logp_difference/mean": 1.6520887613296509, "step": 191, "step_time": 19.024122532035108 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033808479784056544, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.0625, "completions/mean_terminated_length": 17.724138259887695, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0697040185332298, "epoch": 0.00384, "frac_reward_zero_std": 0.0, "grad_norm": 0.3037246763706207, "kl": 0.3097362248227, "learning_rate": 9.41403851423797e-06, "loss": -0.0833, "num_tokens": 8594411.0, "reward": -3.725290298461914e-09, "reward_std": 0.302062451839447, "rewards/rollout_reward_func/mean": -3.725290298461914e-09, "rewards/rollout_reward_func/std": 0.31520602107048035, "sampling/importance_sampling_ratio/max": 2.189652442932129, "sampling/importance_sampling_ratio/mean": 0.4038877487182617, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.3050422668457, "sampling/sampling_logp_difference/mean": 2.2429986000061035, "step": 192, "step_time": 19.290285258975928 }, { "clip_ratio/high_max": 0.00812499993480742, "clip_ratio/high_mean": 0.00406249996740371, "clip_ratio/low_mean": 0.011253720265813172, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015316220233216882, "completions/clipped_ratio": 0.25, "completions/max_length": 42.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 16.90625, "completions/mean_terminated_length": 14.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0547857992351055, "epoch": 0.00386, "frac_reward_zero_std": 0.0, "grad_norm": 0.44255515933036804, "kl": 0.9668968948535621, "learning_rate": 9.406707379266792e-06, "loss": -0.0831, "num_tokens": 8638838.0, "reward": -0.07187500596046448, "reward_std": 0.3655405044555664, "rewards/rollout_reward_func/mean": -0.07187500596046448, "rewards/rollout_reward_func/std": 0.35671836137771606, "sampling/importance_sampling_ratio/max": 2.7170445919036865, "sampling/importance_sampling_ratio/mean": 0.47788316011428833, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.03878402709961, "sampling/sampling_logp_difference/mean": 1.9604331254959106, "step": 193, "step_time": 18.713152611002442 }, { "clip_ratio/high_max": 0.006770833628252149, "clip_ratio/high_mean": 0.0033854168141260743, "clip_ratio/low_mean": 0.003931051644030958, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007316468458157033, "completions/clipped_ratio": 0.21875, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 17.9375, "completions/mean_terminated_length": 14.519999504089355, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9464073572307825, "epoch": 0.00388, "frac_reward_zero_std": 0.0, "grad_norm": 0.17182011902332306, "kl": 0.31812147214077413, "learning_rate": 9.399334605191332e-06, "loss": -0.0754, "num_tokens": 8684867.0, "reward": -0.0624999962747097, "reward_std": 0.3154851794242859, "rewards/rollout_reward_func/mean": -0.0624999962747097, "rewards/rollout_reward_func/std": 0.3180079162120819, "sampling/importance_sampling_ratio/max": 2.239915609359741, "sampling/importance_sampling_ratio/mean": 0.48211193084716797, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.90224075317383, "sampling/sampling_logp_difference/mean": 2.0797314643859863, "step": 194, "step_time": 19.21134512098797 }, { "clip_ratio/high_max": 0.006477272836491466, "clip_ratio/high_mean": 0.003238636418245733, "clip_ratio/low_mean": 0.0059291444486007094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009167780866846442, "completions/clipped_ratio": 0.03125, "completions/max_length": 33.0, "completions/max_terminated_length": 33.0, "completions/mean_length": 17.09375, "completions/mean_terminated_length": 16.612903594970703, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0838046949356794, "epoch": 0.0039, "frac_reward_zero_std": 0.0, "grad_norm": 0.2989274859428406, "kl": 0.2842698711901903, "learning_rate": 9.391920289263475e-06, "loss": 0.0061, "num_tokens": 8729957.0, "reward": -0.02187500149011612, "reward_std": 0.28196409344673157, "rewards/rollout_reward_func/mean": -0.02187500149011612, "rewards/rollout_reward_func/std": 0.2980954349040985, "sampling/importance_sampling_ratio/max": 1.8147433996200562, "sampling/importance_sampling_ratio/mean": 0.3319438695907593, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.23359680175781, "sampling/sampling_logp_difference/mean": 2.083097457885742, "step": 195, "step_time": 18.99095586600015 }, { "clip_ratio/high_max": 0.012332305428571999, "clip_ratio/high_mean": 0.0061661527142859995, "clip_ratio/low_mean": 0.0036122312303632498, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00977838394464925, "completions/clipped_ratio": 0.125, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 16.84375, "completions/mean_terminated_length": 15.285715103149414, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9205640535801649, "epoch": 0.00392, "frac_reward_zero_std": 0.0, "grad_norm": 0.24645204842090607, "kl": 0.510782852768898, "learning_rate": 9.384464529283055e-06, "loss": 0.0365, "num_tokens": 8774640.0, "reward": -0.08749999850988388, "reward_std": 0.2530691921710968, "rewards/rollout_reward_func/mean": -0.08749999850988388, "rewards/rollout_reward_func/std": 0.3414863049983978, "sampling/importance_sampling_ratio/max": 2.0030100345611572, "sampling/importance_sampling_ratio/mean": 0.4139711856842041, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.4198112487793, "sampling/sampling_logp_difference/mean": 1.9357990026474, "step": 196, "step_time": 18.983760470015113 }, { "clip_ratio/high_max": 0.01628917409107089, "clip_ratio/high_mean": 0.008144587045535445, "clip_ratio/low_mean": 0.002366071450524032, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010510658496059477, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.78125, "completions/mean_terminated_length": 17.076923370361328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9927865862846375, "epoch": 0.00394, "frac_reward_zero_std": 0.0, "grad_norm": 0.2933102250099182, "kl": 0.38926855847239494, "learning_rate": 9.376967423596588e-06, "loss": -0.1708, "num_tokens": 8819043.0, "reward": -0.05625000223517418, "reward_std": 0.2846181392669678, "rewards/rollout_reward_func/mean": -0.05625000223517418, "rewards/rollout_reward_func/std": 0.2906472086906433, "sampling/importance_sampling_ratio/max": 2.6422953605651855, "sampling/importance_sampling_ratio/mean": 0.4289810061454773, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.906768798828125, "sampling/sampling_logp_difference/mean": 1.9233078956604004, "step": 197, "step_time": 19.00617637000687 }, { "clip_ratio/high_max": 0.0056925995741039515, "clip_ratio/high_mean": 0.0028462997870519757, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028462997870519757, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.15625, "completions/mean_terminated_length": 16.307693481445312, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0051179267466068, "epoch": 0.00396, "frac_reward_zero_std": 0.0, "grad_norm": 0.21244707703590393, "kl": 0.552051454083994, "learning_rate": 9.369429071095963e-06, "loss": -0.1169, "num_tokens": 8864676.0, "reward": -0.16875000298023224, "reward_std": 0.316331684589386, "rewards/rollout_reward_func/mean": -0.16875000298023224, "rewards/rollout_reward_func/std": 0.32072246074676514, "sampling/importance_sampling_ratio/max": 1.5147030353546143, "sampling/importance_sampling_ratio/mean": 0.30932605266571045, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.902793884277344, "sampling/sampling_logp_difference/mean": 1.9932719469070435, "step": 198, "step_time": 18.906479485012824 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010775862028822303, "completions/clipped_ratio": 0.40625, "completions/max_length": 48.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.03125, "completions/mean_terminated_length": 13.684210777282715, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0247671827673912, "epoch": 0.00398, "frac_reward_zero_std": 0.0, "grad_norm": 0.3196313977241516, "kl": 0.47912247106432915, "learning_rate": 9.361849571217149e-06, "loss": 0.0133, "num_tokens": 8910526.0, "reward": -0.12187499552965164, "reward_std": 0.3292771279811859, "rewards/rollout_reward_func/mean": -0.12187499552965164, "rewards/rollout_reward_func/std": 0.32797953486442566, "sampling/importance_sampling_ratio/max": 1.9539809226989746, "sampling/importance_sampling_ratio/mean": 0.4664402902126312, "sampling/importance_sampling_ratio/min": 5.184804318001823e-44, "sampling/sampling_logp_difference/max": 33.406734466552734, "sampling/sampling_logp_difference/mean": 1.4318922758102417, "step": 199, "step_time": 19.168778519975604 }, { "clip_ratio/high_max": 0.00446998723782599, "clip_ratio/high_mean": 0.002234993618912995, "clip_ratio/low_mean": 0.003160919644869864, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005395913263782859, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.21875, "completions/mean_terminated_length": 17.46428680419922, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1508460156619549, "epoch": 0.004, "frac_reward_zero_std": 0.0, "grad_norm": 0.22622697055339813, "kl": 0.3549641063436866, "learning_rate": 9.35422902393887e-06, "loss": -0.0781, "num_tokens": 8956289.0, "reward": -0.02500000037252903, "reward_std": 0.3335690498352051, "rewards/rollout_reward_func/mean": -0.02500000037252903, "rewards/rollout_reward_func/std": 0.3321484327316284, "sampling/importance_sampling_ratio/max": 1.535462737083435, "sampling/importance_sampling_ratio/mean": 0.32269081473350525, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.94510269165039, "sampling/sampling_logp_difference/mean": 2.5267438888549805, "step": 200, "step_time": 18.92565227500745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0010775862028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010775862028822303, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 16.40625, "completions/mean_terminated_length": 15.407407760620117, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0046967025846243, "epoch": 0.00402, "frac_reward_zero_std": 0.0, "grad_norm": 0.40788522362709045, "kl": 0.25105944834649563, "learning_rate": 9.346567529781298e-06, "loss": -0.0297, "num_tokens": 8999892.0, "reward": 0.0468750074505806, "reward_std": 0.2652738094329834, "rewards/rollout_reward_func/mean": 0.0468750074505806, "rewards/rollout_reward_func/std": 0.3162118196487427, "sampling/importance_sampling_ratio/max": 1.8988533020019531, "sampling/importance_sampling_ratio/mean": 0.534784197807312, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.81298065185547, "sampling/sampling_logp_difference/mean": 1.9666872024536133, "step": 201, "step_time": 18.48339935603144 }, { "clip_ratio/high_max": 0.0036764706019312143, "clip_ratio/high_mean": 0.0018382353009656072, "clip_ratio/low_mean": 0.0018788682646118104, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037171035655774176, "completions/clipped_ratio": 0.125, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 17.71875, "completions/mean_terminated_length": 15.678571701049805, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9284475464373827, "epoch": 0.00404, "frac_reward_zero_std": 0.0, "grad_norm": 0.46082013845443726, "kl": 0.4526989341247827, "learning_rate": 9.338865189804722e-06, "loss": -0.0664, "num_tokens": 9044378.0, "reward": -0.012499997392296791, "reward_std": 0.33273839950561523, "rewards/rollout_reward_func/mean": -0.012499997392296791, "rewards/rollout_reward_func/std": 0.32897692918777466, "sampling/importance_sampling_ratio/max": 2.4459123611450195, "sampling/importance_sampling_ratio/mean": 0.5250406265258789, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.30157470703125, "sampling/sampling_logp_difference/mean": 1.822083830833435, "step": 202, "step_time": 19.449933923009667 }, { "clip_ratio/high_max": 0.014025297947227955, "clip_ratio/high_mean": 0.007012648973613977, "clip_ratio/low_mean": 0.0007621950935572386, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007774844067171216, "completions/clipped_ratio": 0.3125, "completions/max_length": 44.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 20.59375, "completions/mean_terminated_length": 15.272727966308594, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1453194431960583, "epoch": 0.00406, "frac_reward_zero_std": 0.0, "grad_norm": 0.16496488451957703, "kl": 0.19735381356440485, "learning_rate": 9.33112210560821e-06, "loss": -0.0763, "num_tokens": 9089617.0, "reward": -0.12812499701976776, "reward_std": 0.4114035367965698, "rewards/rollout_reward_func/mean": -0.12812499701976776, "rewards/rollout_reward_func/std": 0.4183179438114166, "sampling/importance_sampling_ratio/max": 1.5859662294387817, "sampling/importance_sampling_ratio/mean": 0.34682831168174744, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.82066345214844, "sampling/sampling_logp_difference/mean": 2.305553436279297, "step": 203, "step_time": 19.709699957020348 }, { "clip_ratio/high_max": 0.011332418071106076, "clip_ratio/high_mean": 0.005666209035553038, "clip_ratio/low_mean": 0.004936079611070454, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010602288646623492, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 16.625, "completions/mean_terminated_length": 15.034482955932617, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9094161801040173, "epoch": 0.00408, "frac_reward_zero_std": 0.0, "grad_norm": 0.1922816038131714, "kl": 0.44725408824160695, "learning_rate": 9.323338379328278e-06, "loss": -0.0581, "num_tokens": 9135271.0, "reward": -0.04062500223517418, "reward_std": 0.26215890049934387, "rewards/rollout_reward_func/mean": -0.04062500223517418, "rewards/rollout_reward_func/std": 0.29823067784309387, "sampling/importance_sampling_ratio/max": 1.676526665687561, "sampling/importance_sampling_ratio/mean": 0.4701389968395233, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.46633529663086, "sampling/sampling_logp_difference/mean": 1.8435333967208862, "step": 204, "step_time": 19.102878598016105 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028947368264198303, "completions/clipped_ratio": 0.125, "completions/max_length": 31.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 16.5625, "completions/mean_terminated_length": 16.178571701049805, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0984471067786217, "epoch": 0.0041, "frac_reward_zero_std": 0.0, "grad_norm": 0.2537403702735901, "kl": 0.22938105231150985, "learning_rate": 9.315514113637534e-06, "loss": -0.1352, "num_tokens": 9180459.0, "reward": -0.04062499850988388, "reward_std": 0.29333341121673584, "rewards/rollout_reward_func/mean": -0.04062499850988388, "rewards/rollout_reward_func/std": 0.30676183104515076, "sampling/importance_sampling_ratio/max": 2.026820421218872, "sampling/importance_sampling_ratio/mean": 0.5421896576881409, "sampling/importance_sampling_ratio/min": 3.154322843195163e-42, "sampling/sampling_logp_difference/max": 33.421104431152344, "sampling/sampling_logp_difference/mean": 2.2561745643615723, "step": 205, "step_time": 17.874656807020074 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.002697172632906586, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003774758835788816, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 19.625, "completions/mean_terminated_length": 16.600000381469727, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.060972336679697, "epoch": 0.00412, "frac_reward_zero_std": 0.0, "grad_norm": 0.22254623472690582, "kl": 0.26906318590044975, "learning_rate": 9.30764941174334e-06, "loss": -0.1059, "num_tokens": 9224177.0, "reward": 0.003125004470348358, "reward_std": 0.3449332118034363, "rewards/rollout_reward_func/mean": 0.003125004470348358, "rewards/rollout_reward_func/std": 0.3514801561832428, "sampling/importance_sampling_ratio/max": 1.4809781312942505, "sampling/importance_sampling_ratio/mean": 0.3368404507637024, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.87422561645508, "sampling/sampling_logp_difference/mean": 2.0136590003967285, "step": 206, "step_time": 21.03083078295458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0009191176504828036, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009191176504828036, "completions/clipped_ratio": 0.1875, "completions/max_length": 36.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 16.3125, "completions/mean_terminated_length": 14.307692527770996, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0018754191696644, "epoch": 0.00414, "frac_reward_zero_std": 0.0, "grad_norm": 0.3400401771068573, "kl": 0.20275037363171577, "learning_rate": 9.299744377386424e-06, "loss": -0.0465, "num_tokens": 9268871.0, "reward": -0.03437499701976776, "reward_std": 0.36178743839263916, "rewards/rollout_reward_func/mean": -0.03437499701976776, "rewards/rollout_reward_func/std": 0.346977174282074, "sampling/importance_sampling_ratio/max": 1.9261479377746582, "sampling/importance_sampling_ratio/mean": 0.5168969631195068, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.060546875, "sampling/sampling_logp_difference/mean": 2.1991333961486816, "step": 207, "step_time": 18.378962438961025 }, { "clip_ratio/high_max": 0.022496657678857446, "clip_ratio/high_mean": 0.011248328839428723, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011248328839428723, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.375, "completions/mean_terminated_length": 16.703702926635742, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9921187348663807, "epoch": 0.00416, "frac_reward_zero_std": 0.0, "grad_norm": 0.17583830654621124, "kl": 0.20578415831550956, "learning_rate": 9.291799114839533e-06, "loss": -0.1295, "num_tokens": 9313637.0, "reward": 0.046875, "reward_std": 0.3800668716430664, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.3671901822090149, "sampling/importance_sampling_ratio/max": 1.9304593801498413, "sampling/importance_sampling_ratio/mean": 0.44588184356689453, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.98359298706055, "sampling/sampling_logp_difference/mean": 1.860544204711914, "step": 208, "step_time": 19.510298198991222 }, { "clip_ratio/high_max": 0.015118185430765152, "clip_ratio/high_mean": 0.007559092715382576, "clip_ratio/low_mean": 0.009062499972060323, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01662159222178161, "completions/clipped_ratio": 0.28125, "completions/max_length": 41.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.75, "completions/mean_terminated_length": 15.34782600402832, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3697090037167072, "epoch": 0.00418, "frac_reward_zero_std": 0.0, "grad_norm": 0.21311597526073456, "kl": 0.18966761697083712, "learning_rate": 9.283813728906054e-06, "loss": -0.0909, "num_tokens": 9359836.0, "reward": -0.10000000149011612, "reward_std": 0.3441352844238281, "rewards/rollout_reward_func/mean": -0.10000000149011612, "rewards/rollout_reward_func/std": 0.333118200302124, "sampling/importance_sampling_ratio/max": 1.7086020708084106, "sampling/importance_sampling_ratio/mean": 0.32281118631362915, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.56056213378906, "sampling/sampling_logp_difference/mean": 2.6147665977478027, "step": 209, "step_time": 19.589904231019318 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.005227426707278937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006305012910161167, "completions/clipped_ratio": 0.21875, "completions/max_length": 37.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 15.4375, "completions/mean_terminated_length": 12.679999351501465, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.125982267782092, "epoch": 0.0042, "frac_reward_zero_std": 0.0, "grad_norm": 0.4691476821899414, "kl": 0.25570989307016134, "learning_rate": 9.275788324918624e-06, "loss": -0.0987, "num_tokens": 9405123.0, "reward": -0.08124999701976776, "reward_std": 0.3377065360546112, "rewards/rollout_reward_func/mean": -0.08124999701976776, "rewards/rollout_reward_func/std": 0.3296503722667694, "sampling/importance_sampling_ratio/max": 1.5611060857772827, "sampling/importance_sampling_ratio/mean": 0.47798675298690796, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.10591125488281, "sampling/sampling_logp_difference/mean": 2.260124444961548, "step": 210, "step_time": 18.348617794006714 }, { "clip_ratio/high_max": 0.004903846187517047, "clip_ratio/high_mean": 0.0024519230937585235, "clip_ratio/low_mean": 0.010641664848662913, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013093587942421436, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.84375, "completions/mean_terminated_length": 16.479999542236328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0618420876562595, "epoch": 0.00422, "frac_reward_zero_std": 0.0, "grad_norm": 0.322155237197876, "kl": 0.45941374311223626, "learning_rate": 9.267723008737745e-06, "loss": -0.028, "num_tokens": 9449998.0, "reward": 0.03437500074505806, "reward_std": 0.28858935832977295, "rewards/rollout_reward_func/mean": 0.03437500074505806, "rewards/rollout_reward_func/std": 0.31988847255706787, "sampling/importance_sampling_ratio/max": 1.7847330570220947, "sampling/importance_sampling_ratio/mean": 0.3995518386363983, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.201019287109375, "sampling/sampling_logp_difference/mean": 1.8846921920776367, "step": 211, "step_time": 19.40731012901233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 30.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 14.8125, "completions/mean_terminated_length": 14.214286804199219, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9291577637195587, "epoch": 0.00424, "frac_reward_zero_std": 0.0, "grad_norm": 0.23155982792377472, "kl": 0.3625482995994389, "learning_rate": 9.259617886750395e-06, "loss": -0.0938, "num_tokens": 9493726.0, "reward": 0.006249997764825821, "reward_std": 0.2753341794013977, "rewards/rollout_reward_func/mean": 0.006249997764825821, "rewards/rollout_reward_func/std": 0.29175499081611633, "sampling/importance_sampling_ratio/max": 2.5280025005340576, "sampling/importance_sampling_ratio/mean": 0.6069544553756714, "sampling/importance_sampling_ratio/min": 1.5414283107572988e-44, "sampling/sampling_logp_difference/max": 32.618526458740234, "sampling/sampling_logp_difference/mean": 1.5782983303070068, "step": 212, "step_time": 18.336054659986985 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0026086956495419145, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0045618206495419145, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 15.78125, "completions/mean_terminated_length": 14.041666984558105, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.9031882807612419, "epoch": 0.00426, "frac_reward_zero_std": 0.0, "grad_norm": 0.3379601538181305, "kl": 0.2345649094786495, "learning_rate": 9.251473065868606e-06, "loss": -0.0951, "num_tokens": 9537225.0, "reward": 0.025000005960464478, "reward_std": 0.3232244849205017, "rewards/rollout_reward_func/mean": 0.025000005960464478, "rewards/rollout_reward_func/std": 0.32028213143348694, "sampling/importance_sampling_ratio/max": 1.7842183113098145, "sampling/importance_sampling_ratio/mean": 0.5826365947723389, "sampling/importance_sampling_ratio/min": 4.9690043544958013e-42, "sampling/sampling_logp_difference/max": 33.07731246948242, "sampling/sampling_logp_difference/mean": 1.328566551208496, "step": 213, "step_time": 19.063803950004512 }, { "clip_ratio/high_max": 0.013960353564471006, "clip_ratio/high_mean": 0.006980176782235503, "clip_ratio/low_mean": 0.0020833334419876337, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009063510224223137, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 16.4375, "completions/mean_terminated_length": 14.079999923706055, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1081498712301254, "epoch": 0.00428, "frac_reward_zero_std": 0.0, "grad_norm": 0.38776424527168274, "kl": 0.40600269148126245, "learning_rate": 9.243288653528075e-06, "loss": -0.0494, "num_tokens": 9582271.0, "reward": -0.01875000260770321, "reward_std": 0.2870689928531647, "rewards/rollout_reward_func/mean": -0.01875000260770321, "rewards/rollout_reward_func/std": 0.29450613260269165, "sampling/importance_sampling_ratio/max": 1.7328683137893677, "sampling/importance_sampling_ratio/mean": 0.38680946826934814, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 33.55167770385742, "sampling/sampling_logp_difference/mean": 1.8614587783813477, "step": 214, "step_time": 19.718118824996054 }, { "clip_ratio/high_max": 0.006696428870782256, "clip_ratio/high_mean": 0.003348214435391128, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003348214435391128, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 17.625, "completions/mean_terminated_length": 15.730770111083984, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9871140830218792, "epoch": 0.0043, "frac_reward_zero_std": 0.0, "grad_norm": 0.367491215467453, "kl": 0.17440232424996793, "learning_rate": 9.235064757686733e-06, "loss": -0.1402, "num_tokens": 9625486.0, "reward": -0.00937499850988388, "reward_std": 0.32467764616012573, "rewards/rollout_reward_func/mean": -0.00937499850988388, "rewards/rollout_reward_func/std": 0.35319679975509644, "sampling/importance_sampling_ratio/max": 1.6011875867843628, "sampling/importance_sampling_ratio/mean": 0.522534966468811, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.22354507446289, "sampling/sampling_logp_difference/mean": 1.8239178657531738, "step": 215, "step_time": 18.976297621018603 }, { "clip_ratio/high_max": 0.01131831994280219, "clip_ratio/high_mean": 0.005659159971401095, "clip_ratio/low_mean": 0.0029956427169963717, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008654802688397467, "completions/clipped_ratio": 0.1875, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 20.375, "completions/mean_terminated_length": 19.76923179626465, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3548089507967234, "epoch": 0.00432, "frac_reward_zero_std": 0.0, "grad_norm": 0.4516211450099945, "kl": 0.22819425584748387, "learning_rate": 9.226801486823325e-06, "loss": -0.1542, "num_tokens": 9671021.0, "reward": -0.05000000074505806, "reward_std": 0.31934359669685364, "rewards/rollout_reward_func/mean": -0.05000000074505806, "rewards/rollout_reward_func/std": 0.313152551651001, "sampling/importance_sampling_ratio/max": 2.735166549682617, "sampling/importance_sampling_ratio/mean": 0.32977741956710815, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.2326545715332, "sampling/sampling_logp_difference/mean": 2.7828569412231445, "step": 216, "step_time": 19.804413426987594 }, { "clip_ratio/high_max": 0.00901737972162664, "clip_ratio/high_mean": 0.0062448009848594666, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007402208400890231, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.625, "completions/mean_terminated_length": 16.153846740722656, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.059410797432065, "epoch": 0.00434, "frac_reward_zero_std": 0.0, "grad_norm": 0.4474598169326782, "kl": 0.1906872543040663, "learning_rate": 9.21849894993598e-06, "loss": -0.0879, "num_tokens": 9715523.0, "reward": 0.0031250007450580597, "reward_std": 0.36624854803085327, "rewards/rollout_reward_func/mean": 0.0031250007450580597, "rewards/rollout_reward_func/std": 0.36587002873420715, "sampling/importance_sampling_ratio/max": 2.3677165508270264, "sampling/importance_sampling_ratio/mean": 0.559508204460144, "sampling/importance_sampling_ratio/min": 6.109661304456202e-42, "sampling/sampling_logp_difference/max": 33.90277862548828, "sampling/sampling_logp_difference/mean": 2.03928279876709, "step": 217, "step_time": 18.665221047005616 }, { "clip_ratio/high_max": 0.0026041667442768812, "clip_ratio/high_mean": 0.0013020833721384406, "clip_ratio/low_mean": 0.0028462997870519757, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004148383159190416, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.125, "completions/mean_terminated_length": 15.785715103149414, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1050825379788876, "epoch": 0.00436, "frac_reward_zero_std": 0.0, "grad_norm": 0.4373059868812561, "kl": 0.15060418797656894, "learning_rate": 9.21015725654077e-06, "loss": -0.0093, "num_tokens": 9761162.0, "reward": -0.01562499813735485, "reward_std": 0.29205322265625, "rewards/rollout_reward_func/mean": -0.01562499813735485, "rewards/rollout_reward_func/std": 0.3173574209213257, "sampling/importance_sampling_ratio/max": 2.6367266178131104, "sampling/importance_sampling_ratio/mean": 0.4633912444114685, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.38716125488281, "sampling/sampling_logp_difference/mean": 2.3906078338623047, "step": 218, "step_time": 19.049233790967264 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.0012019231216982007, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002279509324580431, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.1875, "completions/mean_terminated_length": 17.20689582824707, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.057954415678978, "epoch": 0.00438, "frac_reward_zero_std": 0.0, "grad_norm": 0.42218273878097534, "kl": 0.35426637763157487, "learning_rate": 9.201776516670274e-06, "loss": -0.0266, "num_tokens": 9806953.0, "reward": -0.07499999552965164, "reward_std": 0.24980047345161438, "rewards/rollout_reward_func/mean": -0.07499999552965164, "rewards/rollout_reward_func/std": 0.27356666326522827, "sampling/importance_sampling_ratio/max": 1.749214768409729, "sampling/importance_sampling_ratio/mean": 0.47134625911712646, "sampling/importance_sampling_ratio/min": 5.314438438936512e-40, "sampling/sampling_logp_difference/max": 33.12099075317383, "sampling/sampling_logp_difference/mean": 1.6902070045471191, "step": 219, "step_time": 18.568483861992718 }, { "clip_ratio/high_max": 0.005155724007636309, "clip_ratio/high_mean": 0.0025778620038181543, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025778620038181543, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 16.9375, "completions/mean_terminated_length": 15.807692527770996, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0393565818667412, "epoch": 0.0044, "frac_reward_zero_std": 0.0, "grad_norm": 0.40118902921676636, "kl": 0.1926111774519086, "learning_rate": 9.193356840872111e-06, "loss": -0.1029, "num_tokens": 9851609.0, "reward": -0.09687500447034836, "reward_std": 0.35272207856178284, "rewards/rollout_reward_func/mean": -0.09687500447034836, "rewards/rollout_reward_func/std": 0.36232614517211914, "sampling/importance_sampling_ratio/max": 1.6812716722488403, "sampling/importance_sampling_ratio/mean": 0.4908220171928406, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.216346740722656, "sampling/sampling_logp_difference/mean": 1.7205212116241455, "step": 220, "step_time": 18.820035017997725 }, { "clip_ratio/high_max": 0.016249183798208833, "clip_ratio/high_mean": 0.009326515137217939, "clip_ratio/low_mean": 0.003564814804121852, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01289132994133979, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 21.96875, "completions/mean_terminated_length": 20.31999969482422, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1970055475831032, "epoch": 0.00442, "frac_reward_zero_std": 0.0, "grad_norm": 0.20440074801445007, "kl": 0.1434760163538158, "learning_rate": 9.184898340207502e-06, "loss": -0.0092, "num_tokens": 9898670.0, "reward": -0.10312499850988388, "reward_std": 0.3144720792770386, "rewards/rollout_reward_func/mean": -0.10312499850988388, "rewards/rollout_reward_func/std": 0.31055039167404175, "sampling/importance_sampling_ratio/max": 1.5495842695236206, "sampling/importance_sampling_ratio/mean": 0.32359951734542847, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.3328971862793, "sampling/sampling_logp_difference/mean": 2.1067399978637695, "step": 221, "step_time": 20.283040107999113 }, { "clip_ratio/high_max": 0.005558300530537963, "clip_ratio/high_mean": 0.00378721475135535, "clip_ratio/low_mean": 0.0010775862028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00486480095423758, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.4375, "completions/mean_terminated_length": 19.038461685180664, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0149920769035816, "epoch": 0.00444, "frac_reward_zero_std": 0.0, "grad_norm": 0.2843516767024994, "kl": 0.08683872106485069, "learning_rate": 9.176401126249785e-06, "loss": -0.0762, "num_tokens": 9942597.0, "reward": 0.04375000298023224, "reward_std": 0.3565678596496582, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.36361756920814514, "sampling/importance_sampling_ratio/max": 1.4515455961227417, "sampling/importance_sampling_ratio/mean": 0.4929732084274292, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.62422561645508, "sampling/sampling_logp_difference/mean": 1.815850853919983, "step": 222, "step_time": 18.875118393014418 }, { "clip_ratio/high_max": 0.005476190475746989, "clip_ratio/high_mean": 0.0027380952378734946, "clip_ratio/low_mean": 0.005069633305538446, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007807728601619601, "completions/clipped_ratio": 0.34375, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 21.96875, "completions/mean_terminated_length": 20.5238094329834, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.2016233429312706, "epoch": 0.00446, "frac_reward_zero_std": 0.0, "grad_norm": 0.31329289078712463, "kl": 0.18568478955421597, "learning_rate": 9.167865311082957e-06, "loss": -0.2061, "num_tokens": 9987786.0, "reward": -0.125, "reward_std": 0.3745938241481781, "rewards/rollout_reward_func/mean": -0.125, "rewards/rollout_reward_func/std": 0.36010751128196716, "sampling/importance_sampling_ratio/max": 1.3831431865692139, "sampling/importance_sampling_ratio/mean": 0.31033265590667725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.7425537109375, "sampling/sampling_logp_difference/mean": 2.184202194213867, "step": 223, "step_time": 20.26085782899463 }, { "clip_ratio/high_max": 0.004903846187517047, "clip_ratio/high_mean": 0.0024519230937585235, "clip_ratio/low_mean": 0.0022265624720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004678485565818846, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.125, "completions/mean_terminated_length": 16.399999618530273, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9634398370981216, "epoch": 0.00448, "frac_reward_zero_std": 0.0, "grad_norm": 0.42026564478874207, "kl": 0.42919028946198523, "learning_rate": 9.159291007300193e-06, "loss": -0.2357, "num_tokens": 10032341.0, "reward": -0.12187499552965164, "reward_std": 0.3213362693786621, "rewards/rollout_reward_func/mean": -0.12187499552965164, "rewards/rollout_reward_func/std": 0.3087272346019745, "sampling/importance_sampling_ratio/max": 1.9575200080871582, "sampling/importance_sampling_ratio/mean": 0.5027186870574951, "sampling/importance_sampling_ratio/min": 2.2238606628834847e-42, "sampling/sampling_logp_difference/max": 33.25479507446289, "sampling/sampling_logp_difference/mean": 1.621032953262329, "step": 224, "step_time": 18.954431897000177 }, { "clip_ratio/high_max": 0.010496633127331734, "clip_ratio/high_mean": 0.005248316563665867, "clip_ratio/low_mean": 0.0007812500116415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006029566575307399, "completions/clipped_ratio": 0.25, "completions/max_length": 48.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 20.5625, "completions/mean_terminated_length": 20.08333396911621, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0510635823011398, "epoch": 0.0045, "frac_reward_zero_std": 0.0, "grad_norm": 0.40186604857444763, "kl": 0.20607586856931448, "learning_rate": 9.150678328002352e-06, "loss": -0.1249, "num_tokens": 10075504.0, "reward": 0.015625, "reward_std": 0.3437649607658386, "rewards/rollout_reward_func/mean": 0.015625, "rewards/rollout_reward_func/std": 0.3637700080871582, "sampling/importance_sampling_ratio/max": 2.854978561401367, "sampling/importance_sampling_ratio/mean": 0.5793967843055725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.63984680175781, "sampling/sampling_logp_difference/mean": 1.7840569019317627, "step": 225, "step_time": 19.684994579962222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0041082974057644606, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0041082974057644606, "completions/clipped_ratio": 0.34375, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 19.28125, "completions/mean_terminated_length": 16.095237731933594, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1031633988022804, "epoch": 0.00452, "frac_reward_zero_std": 0.0, "grad_norm": 0.2631945013999939, "kl": 0.22039789578411728, "learning_rate": 9.1420273867965e-06, "loss": -0.1269, "num_tokens": 10120828.0, "reward": -0.171875, "reward_std": 0.3528553247451782, "rewards/rollout_reward_func/mean": -0.171875, "rewards/rollout_reward_func/std": 0.3438251316547394, "sampling/importance_sampling_ratio/max": 1.5066723823547363, "sampling/importance_sampling_ratio/mean": 0.4202766418457031, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.742549896240234, "sampling/sampling_logp_difference/mean": 1.740734338760376, "step": 226, "step_time": 19.767570211013663 }, { "clip_ratio/high_max": 0.01305708778090775, "clip_ratio/high_mean": 0.006528543890453875, "clip_ratio/low_mean": 0.0013020833721384406, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007830627262592316, "completions/clipped_ratio": 0.21875, "completions/max_length": 45.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 19.3125, "completions/mean_terminated_length": 17.600000381469727, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.13470858335495, "epoch": 0.00454, "frac_reward_zero_std": 0.0, "grad_norm": 0.30796363949775696, "kl": 0.17560013523325324, "learning_rate": 9.133338297794396e-06, "loss": -0.0849, "num_tokens": 10165890.0, "reward": 0.02187500149011612, "reward_std": 0.32205355167388916, "rewards/rollout_reward_func/mean": 0.02187500149011612, "rewards/rollout_reward_func/std": 0.3338285982608795, "sampling/importance_sampling_ratio/max": 2.975538730621338, "sampling/importance_sampling_ratio/mean": 0.4846216142177582, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.79341125488281, "sampling/sampling_logp_difference/mean": 2.217421531677246, "step": 227, "step_time": 18.844042170007015 }, { "clip_ratio/high_max": 0.01024247519671917, "clip_ratio/high_mean": 0.005121237598359585, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005121237598359585, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.25, "completions/mean_terminated_length": 18.33333396911621, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.061973188072443, "epoch": 0.00456, "frac_reward_zero_std": 0.0, "grad_norm": 0.3378071188926697, "kl": 0.14192362898029387, "learning_rate": 9.124611175610998e-06, "loss": -0.1613, "num_tokens": 10210258.0, "reward": -0.0937500074505806, "reward_std": 0.36999696493148804, "rewards/rollout_reward_func/mean": -0.0937500074505806, "rewards/rollout_reward_func/std": 0.3732486069202423, "sampling/importance_sampling_ratio/max": 1.561163067817688, "sampling/importance_sampling_ratio/mean": 0.4313727021217346, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.547393798828125, "sampling/sampling_logp_difference/mean": 2.0933732986450195, "step": 228, "step_time": 20.20439572200121 }, { "clip_ratio/high_max": 0.004583333386108279, "clip_ratio/high_mean": 0.0022916666930541396, "clip_ratio/low_mean": 0.005627185455523431, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007918852148577571, "completions/clipped_ratio": 0.21875, "completions/max_length": 39.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 20.15625, "completions/mean_terminated_length": 18.520000457763672, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.28002317994833, "epoch": 0.00458, "frac_reward_zero_std": 0.0, "grad_norm": 0.30278852581977844, "kl": 0.13582363538444042, "learning_rate": 9.11584613536295e-06, "loss": -0.0959, "num_tokens": 10254931.0, "reward": 0.0625, "reward_std": 0.30551812052726746, "rewards/rollout_reward_func/mean": 0.0625, "rewards/rollout_reward_func/std": 0.32003024220466614, "sampling/importance_sampling_ratio/max": 1.4989217519760132, "sampling/importance_sampling_ratio/mean": 0.34067994356155396, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.7983283996582, "sampling/sampling_logp_difference/mean": 2.405669689178467, "step": 229, "step_time": 18.735050243980368 }, { "clip_ratio/high_max": 0.0059523810632526875, "clip_ratio/high_mean": 0.0029761905316263437, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029761905316263437, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.96875, "completions/mean_terminated_length": 15.960000038146973, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0735657699406147, "epoch": 0.0046, "frac_reward_zero_std": 0.0, "grad_norm": 0.2432541847229004, "kl": 0.09910631366074085, "learning_rate": 9.107043292667056e-06, "loss": -0.1285, "num_tokens": 10299613.0, "reward": 0.04062500223517418, "reward_std": 0.337123304605484, "rewards/rollout_reward_func/mean": 0.04062500223517418, "rewards/rollout_reward_func/std": 0.36087650060653687, "sampling/importance_sampling_ratio/max": 1.5774650573730469, "sampling/importance_sampling_ratio/mean": 0.49464020133018494, "sampling/importance_sampling_ratio/min": 2.802596928649634e-44, "sampling/sampling_logp_difference/max": 33.164424896240234, "sampling/sampling_logp_difference/mean": 1.8837339878082275, "step": 230, "step_time": 19.25196506199427 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 18.375, "completions/mean_terminated_length": 16.5, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9458989594131708, "epoch": 0.00462, "frac_reward_zero_std": 0.0, "grad_norm": 0.2755776345729828, "kl": 0.39635168947279453, "learning_rate": 9.098202763638763e-06, "loss": -0.0675, "num_tokens": 10342848.0, "reward": -0.07500000298023224, "reward_std": 0.30783963203430176, "rewards/rollout_reward_func/mean": -0.07500000298023224, "rewards/rollout_reward_func/std": 0.3069096803665161, "sampling/importance_sampling_ratio/max": 1.6200991868972778, "sampling/importance_sampling_ratio/mean": 0.5293498635292053, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.15676498413086, "sampling/sampling_logp_difference/mean": 1.5971441268920898, "step": 231, "step_time": 19.36647469997115 }, { "clip_ratio/high_max": 0.027604366187006235, "clip_ratio/high_mean": 0.01641087862662971, "clip_ratio/low_mean": 0.0010775862028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01748846482951194, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.96875, "completions/mean_terminated_length": 17.416667938232422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0940434653311968, "epoch": 0.00464, "frac_reward_zero_std": 0.0, "grad_norm": 0.40003567934036255, "kl": 0.16758476686663926, "learning_rate": 9.089324664890625e-06, "loss": -0.0732, "num_tokens": 10388531.0, "reward": -0.02812499925494194, "reward_std": 0.2615458369255066, "rewards/rollout_reward_func/mean": -0.02812499925494194, "rewards/rollout_reward_func/std": 0.25429773330688477, "sampling/importance_sampling_ratio/max": 2.17971134185791, "sampling/importance_sampling_ratio/mean": 0.47126060724258423, "sampling/importance_sampling_ratio/min": 2.6624670822171524e-44, "sampling/sampling_logp_difference/max": 33.425697326660156, "sampling/sampling_logp_difference/mean": 2.1090188026428223, "step": 232, "step_time": 18.410879682021914 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.0038723776815459132, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0049499638844281435, "completions/clipped_ratio": 0.25, "completions/max_length": 45.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.75, "completions/mean_terminated_length": 17.95833396911621, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9721126817166805, "epoch": 0.00466, "frac_reward_zero_std": 0.0, "grad_norm": 0.25820392370224, "kl": 0.15019001276232302, "learning_rate": 9.080409113530767e-06, "loss": -0.1119, "num_tokens": 10433639.0, "reward": -0.02499999850988388, "reward_std": 0.3116064667701721, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.32229018211364746, "sampling/importance_sampling_ratio/max": 1.4981862306594849, "sampling/importance_sampling_ratio/mean": 0.49987557530403137, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.32011032104492, "sampling/sampling_logp_difference/mean": 1.8110487461090088, "step": 233, "step_time": 20.26789752599143 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.0020121950656175613, "clip_ratio/low_min": 0.0015243901871144772, "clip_ratio/region_mean": 0.0030897812684997916, "completions/clipped_ratio": 0.3125, "completions/max_length": 48.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 21.84375, "completions/mean_terminated_length": 18.772727966308594, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.042544923722744, "epoch": 0.00468, "frac_reward_zero_std": 0.0, "grad_norm": 0.2996368706226349, "kl": 0.09935617703013122, "learning_rate": 9.071456227161335e-06, "loss": -0.224, "num_tokens": 10478267.0, "reward": -0.03750000149011612, "reward_std": 0.355131059885025, "rewards/rollout_reward_func/mean": -0.03750000149011612, "rewards/rollout_reward_func/std": 0.35988351702690125, "sampling/importance_sampling_ratio/max": 2.6025032997131348, "sampling/importance_sampling_ratio/mean": 0.46678823232650757, "sampling/importance_sampling_ratio/min": 5.275888718182936e-42, "sampling/sampling_logp_difference/max": 34.028717041015625, "sampling/sampling_logp_difference/mean": 1.6063168048858643, "step": 234, "step_time": 19.77734783901542 }, { "clip_ratio/high_max": 0.011128364596515894, "clip_ratio/high_mean": 0.005564182298257947, "clip_ratio/low_mean": 0.0018382353009656072, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007402417599223554, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.90625, "completions/mean_terminated_length": 18.25, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0799154210835695, "epoch": 0.0047, "frac_reward_zero_std": 0.0, "grad_norm": 0.47023484110832214, "kl": 0.23605791036970913, "learning_rate": 9.062466123876954e-06, "loss": -0.1204, "num_tokens": 10524009.0, "reward": -0.12812501192092896, "reward_std": 0.3318510949611664, "rewards/rollout_reward_func/mean": -0.12812501192092896, "rewards/rollout_reward_func/std": 0.3205181062221527, "sampling/importance_sampling_ratio/max": 2.1841816902160645, "sampling/importance_sampling_ratio/mean": 0.40914076566696167, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.14448547363281, "sampling/sampling_logp_difference/mean": 1.9326393604278564, "step": 235, "step_time": 20.176028066009167 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0032875738106667995, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004646269604563713, "completions/clipped_ratio": 0.375, "completions/max_length": 48.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 20.46875, "completions/mean_terminated_length": 17.30000114440918, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.029142800718546, "epoch": 0.00472, "frac_reward_zero_std": 0.0, "grad_norm": 0.2966567873954773, "kl": 0.19103379163425416, "learning_rate": 9.053438922263165e-06, "loss": -0.1491, "num_tokens": 10569762.0, "reward": -0.18125000596046448, "reward_std": 0.34080711007118225, "rewards/rollout_reward_func/mean": -0.18125000596046448, "rewards/rollout_reward_func/std": 0.3440156579017639, "sampling/importance_sampling_ratio/max": 1.4320212602615356, "sampling/importance_sampling_ratio/mean": 0.4805031418800354, "sampling/importance_sampling_ratio/min": 2.1579996350602183e-43, "sampling/sampling_logp_difference/max": 33.3831787109375, "sampling/sampling_logp_difference/mean": 1.7544058561325073, "step": 236, "step_time": 19.29002596398641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.21875, "completions/mean_terminated_length": 16.26923179626465, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9722066447138786, "epoch": 0.00474, "frac_reward_zero_std": 0.0, "grad_norm": 0.27551791071891785, "kl": 0.30932296020910144, "learning_rate": 9.044374741394859e-06, "loss": 0.017, "num_tokens": 10614250.0, "reward": 0.05937500298023224, "reward_std": 0.2905290722846985, "rewards/rollout_reward_func/mean": 0.05937500298023224, "rewards/rollout_reward_func/std": 0.32114651799201965, "sampling/importance_sampling_ratio/max": 1.4464342594146729, "sampling/importance_sampling_ratio/mean": 0.478466272354126, "sampling/importance_sampling_ratio/min": 1.3452465257518244e-42, "sampling/sampling_logp_difference/max": 32.59425354003906, "sampling/sampling_logp_difference/mean": 1.9226669073104858, "step": 237, "step_time": 18.427288291990408 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0009191176504828036, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022778133279643953, "completions/clipped_ratio": 0.34375, "completions/max_length": 60.0, "completions/max_terminated_length": 60.0, "completions/mean_length": 20.46875, "completions/mean_terminated_length": 18.5238094329834, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9175697974860668, "epoch": 0.00476, "frac_reward_zero_std": 0.0, "grad_norm": 0.5641992092132568, "kl": 0.172552902251482, "learning_rate": 9.035273700834707e-06, "loss": -0.1826, "num_tokens": 10657187.0, "reward": 0.02500000409781933, "reward_std": 0.35283973813056946, "rewards/rollout_reward_func/mean": 0.02500000409781933, "rewards/rollout_reward_func/std": 0.35103532671928406, "sampling/importance_sampling_ratio/max": 2.8690807819366455, "sampling/importance_sampling_ratio/mean": 0.6322405934333801, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.591033935546875, "sampling/sampling_logp_difference/mean": 1.4247468709945679, "step": 238, "step_time": 20.124013836990343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016447368543595076, "completions/clipped_ratio": 0.21875, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 19.0, "completions/mean_terminated_length": 16.399999618530273, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9934881273657084, "epoch": 0.00478, "frac_reward_zero_std": 0.0, "grad_norm": 0.3968384265899658, "kl": 0.3506955374032259, "learning_rate": 9.026135920631592e-06, "loss": -0.1322, "num_tokens": 10701657.0, "reward": 0.05625000223517418, "reward_std": 0.22946742177009583, "rewards/rollout_reward_func/mean": 0.05625000223517418, "rewards/rollout_reward_func/std": 0.25392085313796997, "sampling/importance_sampling_ratio/max": 2.5679445266723633, "sampling/importance_sampling_ratio/mean": 0.5426125526428223, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.10134506225586, "sampling/sampling_logp_difference/mean": 2.018400192260742, "step": 239, "step_time": 18.84127541203634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.5, "completions/mean_terminated_length": 17.928571701049805, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9805006142705679, "epoch": 0.0048, "frac_reward_zero_std": 0.0, "grad_norm": 0.5140879154205322, "kl": 0.14661221485584974, "learning_rate": 9.016961521319006e-06, "loss": -0.003, "num_tokens": 10745605.0, "reward": 0.07187500596046448, "reward_std": 0.28952735662460327, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.2986360192298889, "sampling/importance_sampling_ratio/max": 1.8008453845977783, "sampling/importance_sampling_ratio/mean": 0.5476291179656982, "sampling/importance_sampling_ratio/min": 4.344025239406933e-43, "sampling/sampling_logp_difference/max": 33.53618621826172, "sampling/sampling_logp_difference/mean": 1.752831220626831, "step": 240, "step_time": 19.516334643005393 }, { "clip_ratio/high_max": 0.006250000325962901, "clip_ratio/high_mean": 0.0031250001629814506, "clip_ratio/low_mean": 0.0008223684271797538, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003947368590161204, "completions/clipped_ratio": 0.25, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 20.40625, "completions/mean_terminated_length": 20.041667938232422, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9748721029609442, "epoch": 0.00482, "frac_reward_zero_std": 0.0, "grad_norm": 0.43303194642066956, "kl": 0.27742003300227225, "learning_rate": 9.00775062391348e-06, "loss": -0.1941, "num_tokens": 10789743.0, "reward": 0.012500003911554813, "reward_std": 0.35408124327659607, "rewards/rollout_reward_func/mean": 0.012500003911554813, "rewards/rollout_reward_func/std": 0.3405403792858124, "sampling/importance_sampling_ratio/max": 2.593531608581543, "sampling/importance_sampling_ratio/mean": 0.48796194791793823, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.317447662353516, "sampling/sampling_logp_difference/mean": 1.8811928033828735, "step": 241, "step_time": 18.995850215957034 }, { "clip_ratio/high_max": 0.005759804043918848, "clip_ratio/high_mean": 0.002879902021959424, "clip_ratio/low_mean": 0.0024074073880910873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005287309410050511, "completions/clipped_ratio": 0.3125, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 20.90625, "completions/mean_terminated_length": 19.772727966308594, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1169246286153793, "epoch": 0.00484, "frac_reward_zero_std": 0.0, "grad_norm": 0.2568773925304413, "kl": 0.09387251117732376, "learning_rate": 8.998503349912977e-06, "loss": -0.158, "num_tokens": 10835160.0, "reward": -0.140625, "reward_std": 0.33940085768699646, "rewards/rollout_reward_func/mean": -0.140625, "rewards/rollout_reward_func/std": 0.3499855697154999, "sampling/importance_sampling_ratio/max": 2.4417426586151123, "sampling/importance_sampling_ratio/mean": 0.4605308771133423, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.3732795715332, "sampling/sampling_logp_difference/mean": 1.9571356773376465, "step": 242, "step_time": 20.290821814982337 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024074073880910873, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.71875, "completions/mean_terminated_length": 18.479999542236328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.090790793299675, "epoch": 0.00486, "frac_reward_zero_std": 0.0, "grad_norm": 0.31743940711021423, "kl": 0.4117138162255287, "learning_rate": 8.989219821295294e-06, "loss": -0.0541, "num_tokens": 10881190.0, "reward": -0.015625, "reward_std": 0.2743155360221863, "rewards/rollout_reward_func/mean": -0.015625, "rewards/rollout_reward_func/std": 0.2852382957935333, "sampling/importance_sampling_ratio/max": 2.636380672454834, "sampling/importance_sampling_ratio/mean": 0.4032032787799835, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.32735061645508, "sampling/sampling_logp_difference/mean": 1.9510608911514282, "step": 243, "step_time": 20.67708306100394 }, { "clip_ratio/high_max": 0.010148601606488228, "clip_ratio/high_mean": 0.005074300803244114, "clip_ratio/low_mean": 0.0017025089473463595, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006776809750590473, "completions/clipped_ratio": 0.28125, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 19.78125, "completions/mean_terminated_length": 18.0, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0327265933156013, "epoch": 0.00488, "frac_reward_zero_std": 0.0, "grad_norm": 0.24582776427268982, "kl": 0.1832236306509003, "learning_rate": 8.979900160516447e-06, "loss": -0.1218, "num_tokens": 10925698.0, "reward": -0.02187499776482582, "reward_std": 0.2719929814338684, "rewards/rollout_reward_func/mean": -0.02187499776482582, "rewards/rollout_reward_func/std": 0.331890344619751, "sampling/importance_sampling_ratio/max": 2.530608892440796, "sampling/importance_sampling_ratio/mean": 0.4935498535633087, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.12886428833008, "sampling/sampling_logp_difference/mean": 1.9979060888290405, "step": 244, "step_time": 19.546862021961715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004882812616415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004882812616415322, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.96875, "completions/mean_terminated_length": 20.461538314819336, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0988213941454887, "epoch": 0.0049, "frac_reward_zero_std": 0.0, "grad_norm": 0.3310163617134094, "kl": 0.33056584757287055, "learning_rate": 8.970544490509064e-06, "loss": -0.181, "num_tokens": 10971115.0, "reward": -0.12187500298023224, "reward_std": 0.36343348026275635, "rewards/rollout_reward_func/mean": -0.12187500298023224, "rewards/rollout_reward_func/std": 0.3580722510814667, "sampling/importance_sampling_ratio/max": 1.3248132467269897, "sampling/importance_sampling_ratio/mean": 0.37946993112564087, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.132999420166016, "sampling/sampling_logp_difference/mean": 1.8642349243164062, "step": 245, "step_time": 20.660230664987466 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0009765625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022265624720603228, "completions/clipped_ratio": 0.375, "completions/max_length": 43.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 20.34375, "completions/mean_terminated_length": 18.5, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0610592253506184, "epoch": 0.00492, "frac_reward_zero_std": 0.0, "grad_norm": 0.29104283452033997, "kl": 0.1279663247987628, "learning_rate": 8.961152934680752e-06, "loss": -0.1118, "num_tokens": 11016632.0, "reward": -0.13437500596046448, "reward_std": 0.346807062625885, "rewards/rollout_reward_func/mean": -0.13437500596046448, "rewards/rollout_reward_func/std": 0.35067617893218994, "sampling/importance_sampling_ratio/max": 2.45784592628479, "sampling/importance_sampling_ratio/mean": 0.5176913738250732, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.813026428222656, "sampling/sampling_logp_difference/mean": 1.7751078605651855, "step": 246, "step_time": 19.475708887999645 }, { "clip_ratio/high_max": 0.0026041667442768812, "clip_ratio/high_mean": 0.0013020833721384406, "clip_ratio/low_mean": 0.0008928571478463709, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021949405199848115, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 19.71875, "completions/mean_terminated_length": 16.15999984741211, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0358009282499552, "epoch": 0.00494, "frac_reward_zero_std": 0.0, "grad_norm": 0.29616162180900574, "kl": 0.31986318062990904, "learning_rate": 8.951725616912489e-06, "loss": -0.1805, "num_tokens": 11060302.0, "reward": -0.02499999850988388, "reward_std": 0.3820613920688629, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.3724374771118164, "sampling/importance_sampling_ratio/max": 1.6031543016433716, "sampling/importance_sampling_ratio/mean": 0.4347883462905884, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.89448165893555, "sampling/sampling_logp_difference/mean": 1.824547290802002, "step": 247, "step_time": 20.75120705399604 }, { "clip_ratio/high_max": 0.012183330720290542, "clip_ratio/high_mean": 0.006091665360145271, "clip_ratio/low_mean": 0.0027608084492385387, "clip_ratio/low_min": 0.0022321429569274187, "clip_ratio/region_mean": 0.00885247380938381, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 20.3125, "completions/mean_terminated_length": 18.279998779296875, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1608650386333466, "epoch": 0.00496, "frac_reward_zero_std": 0.0, "grad_norm": 0.29003840684890747, "kl": 0.13050074421335012, "learning_rate": 8.942262661556962e-06, "loss": -0.0991, "num_tokens": 11105290.0, "reward": -0.02499999850988388, "reward_std": 0.33385300636291504, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.3311758041381836, "sampling/importance_sampling_ratio/max": 2.4017674922943115, "sampling/importance_sampling_ratio/mean": 0.38827401399612427, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.37258529663086, "sampling/sampling_logp_difference/mean": 2.3067169189453125, "step": 248, "step_time": 18.717402938957093 }, { "clip_ratio/high_max": 0.0026041667442768812, "clip_ratio/high_mean": 0.0013020833721384406, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013020833721384406, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 18.34375, "completions/mean_terminated_length": 17.125, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9260947369039059, "epoch": 0.00498, "frac_reward_zero_std": 0.0, "grad_norm": 0.3395824730396271, "kl": 0.1076100766658783, "learning_rate": 8.932764193436953e-06, "loss": -0.0532, "num_tokens": 11150394.0, "reward": 0.012500002980232239, "reward_std": 0.2993730902671814, "rewards/rollout_reward_func/mean": 0.012500002980232239, "rewards/rollout_reward_func/std": 0.2926519513130188, "sampling/importance_sampling_ratio/max": 1.400244116783142, "sampling/importance_sampling_ratio/mean": 0.44790011644363403, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.754940032958984, "sampling/sampling_logp_difference/mean": 1.7658206224441528, "step": 249, "step_time": 18.584879180969438 }, { "clip_ratio/high_max": 0.0048363097012043, "clip_ratio/high_mean": 0.00241815485060215, "clip_ratio/low_mean": 0.0021936576813459396, "clip_ratio/low_min": 0.0021551724057644606, "clip_ratio/region_mean": 0.00461181253194809, "completions/clipped_ratio": 0.28125, "completions/max_length": 48.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 20.625, "completions/mean_terminated_length": 17.7391300201416, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9719193167984486, "epoch": 0.005, "frac_reward_zero_std": 0.0, "grad_norm": 0.21912626922130585, "kl": 0.09295413736253977, "learning_rate": 8.923230337843675e-06, "loss": -0.1286, "num_tokens": 11196410.0, "reward": -0.08749999105930328, "reward_std": 0.3264467120170593, "rewards/rollout_reward_func/mean": -0.08749999105930328, "rewards/rollout_reward_func/std": 0.32503101229667664, "sampling/importance_sampling_ratio/max": 1.3643770217895508, "sampling/importance_sampling_ratio/mean": 0.4151889979839325, "sampling/importance_sampling_ratio/min": 1.0860063098517332e-42, "sampling/sampling_logp_difference/max": 33.35444259643555, "sampling/sampling_logp_difference/mean": 1.7246878147125244, "step": 250, "step_time": 21.171540163020836 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0012019231216982007, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024519230937585235, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.71875, "completions/mean_terminated_length": 16.961538314819336, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9706812668591738, "epoch": 0.00502, "frac_reward_zero_std": 0.0, "grad_norm": 0.38334354758262634, "kl": 0.21251422143541276, "learning_rate": 8.913661220535132e-06, "loss": -0.0468, "num_tokens": 11240882.0, "reward": 0.09999999403953552, "reward_std": 0.2759665548801422, "rewards/rollout_reward_func/mean": 0.09999999403953552, "rewards/rollout_reward_func/std": 0.2839809060096741, "sampling/importance_sampling_ratio/max": 1.3594410419464111, "sampling/importance_sampling_ratio/mean": 0.47297173738479614, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.78177261352539, "sampling/sampling_logp_difference/mean": 1.8561720848083496, "step": 251, "step_time": 18.62089159101015 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.0010080644860863686, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024285190738737583, "completions/clipped_ratio": 0.25, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 21.28125, "completions/mean_terminated_length": 19.08333396911621, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0441020596772432, "epoch": 0.00504, "frac_reward_zero_std": 0.0, "grad_norm": 0.211502805352211, "kl": 0.07651985413394868, "learning_rate": 8.904056967734447e-06, "loss": -0.094, "num_tokens": 11286861.0, "reward": -0.09687499701976776, "reward_std": 0.34267866611480713, "rewards/rollout_reward_func/mean": -0.09687499701976776, "rewards/rollout_reward_func/std": 0.3364754319190979, "sampling/importance_sampling_ratio/max": 1.1460727453231812, "sampling/importance_sampling_ratio/mean": 0.3707272708415985, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.466346740722656, "sampling/sampling_logp_difference/mean": 1.8628878593444824, "step": 252, "step_time": 21.45285846198385 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012499999720603228, "completions/clipped_ratio": 0.15625, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 18.28125, "completions/mean_terminated_length": 17.037036895751953, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9086448214948177, "epoch": 0.00506, "frac_reward_zero_std": 0.0, "grad_norm": 0.23674704134464264, "kl": 0.16827611788176, "learning_rate": 8.894417706128209e-06, "loss": -0.1584, "num_tokens": 11330537.0, "reward": 0.14375001192092896, "reward_std": 0.263245165348053, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.263888418674469, "sampling/importance_sampling_ratio/max": 2.588695526123047, "sampling/importance_sampling_ratio/mean": 0.6308994293212891, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.076393127441406, "sampling/sampling_logp_difference/mean": 1.7857478857040405, "step": 253, "step_time": 19.080299084016588 }, { "clip_ratio/high_max": 0.0073363096453249454, "clip_ratio/high_mean": 0.0036681548226624727, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036681548226624727, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 18.0, "completions/mean_terminated_length": 16.88888931274414, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9671693816781044, "epoch": 0.00508, "frac_reward_zero_std": 0.0, "grad_norm": 0.3426640033721924, "kl": 0.24782740999944508, "learning_rate": 8.884743562864792e-06, "loss": -0.1173, "num_tokens": 11374405.0, "reward": 0.0312500074505806, "reward_std": 0.29943159222602844, "rewards/rollout_reward_func/mean": 0.0312500074505806, "rewards/rollout_reward_func/std": 0.2999328076839447, "sampling/importance_sampling_ratio/max": 2.007253646850586, "sampling/importance_sampling_ratio/mean": 0.5716434717178345, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.82734298706055, "sampling/sampling_logp_difference/mean": 1.7906734943389893, "step": 254, "step_time": 18.830468256026506 }, { "clip_ratio/high_max": 0.004464285913854837, "clip_ratio/high_mean": 0.0022321429569274187, "clip_ratio/low_mean": 0.0042053556535393, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006437498610466719, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 17.71875, "completions/mean_terminated_length": 16.959999084472656, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.060914508998394, "epoch": 0.0051, "frac_reward_zero_std": 0.0, "grad_norm": 0.4681198298931122, "kl": 0.3667053112294525, "learning_rate": 8.875034665552686e-06, "loss": -0.0419, "num_tokens": 11419515.0, "reward": -0.09375, "reward_std": 0.3039851188659668, "rewards/rollout_reward_func/mean": -0.09375, "rewards/rollout_reward_func/std": 0.30579087138175964, "sampling/importance_sampling_ratio/max": 1.7764647006988525, "sampling/importance_sampling_ratio/mean": 0.498703271150589, "sampling/importance_sampling_ratio/min": 7.917336323435216e-43, "sampling/sampling_logp_difference/max": 33.581993103027344, "sampling/sampling_logp_difference/mean": 1.607851505279541, "step": 255, "step_time": 19.836224969039904 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036525975447148085, "completions/clipped_ratio": 0.15625, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 18.53125, "completions/mean_terminated_length": 19.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9979402646422386, "epoch": 0.00512, "frac_reward_zero_std": 0.0, "grad_norm": 0.26634207367897034, "kl": 0.6210615115705878, "learning_rate": 8.865291142258813e-06, "loss": -0.1379, "num_tokens": 11463869.0, "reward": 0.006250001490116119, "reward_std": 0.33977386355400085, "rewards/rollout_reward_func/mean": 0.006250001490116119, "rewards/rollout_reward_func/std": 0.32719483971595764, "sampling/importance_sampling_ratio/max": 1.2936283349990845, "sampling/importance_sampling_ratio/mean": 0.43942368030548096, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.70698928833008, "sampling/sampling_logp_difference/mean": 1.6284617185592651, "step": 256, "step_time": 19.093404391009244 }, { "clip_ratio/high_max": 0.003289473708719015, "clip_ratio/high_mean": 0.0016447368543595076, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016447368543595076, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 21.53125, "completions/mean_terminated_length": 20.30434799194336, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1427328698337078, "epoch": 0.00514, "frac_reward_zero_std": 0.0, "grad_norm": 0.3254883289337158, "kl": 0.16724058741237968, "learning_rate": 8.855513121506826e-06, "loss": -0.0891, "num_tokens": 11510595.0, "reward": -0.18125000596046448, "reward_std": 0.2842714786529541, "rewards/rollout_reward_func/mean": -0.18125000596046448, "rewards/rollout_reward_func/std": 0.27990493178367615, "sampling/importance_sampling_ratio/max": 1.4568328857421875, "sampling/importance_sampling_ratio/mean": 0.362612247467041, "sampling/importance_sampling_ratio/min": 3.279038406520072e-43, "sampling/sampling_logp_difference/max": 33.76389694213867, "sampling/sampling_logp_difference/mean": 1.9058022499084473, "step": 257, "step_time": 20.203471356042428 }, { "clip_ratio/high_max": 0.0047186610754579306, "clip_ratio/high_mean": 0.0023593305377289653, "clip_ratio/low_mean": 0.0010416667209938169, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003400997258722782, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.875, "completions/mean_terminated_length": 17.478261947631836, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9263858944177628, "epoch": 0.00516, "frac_reward_zero_std": 0.0, "grad_norm": 0.3497600853443146, "kl": 0.0908962613902986, "learning_rate": 8.845700732275435e-06, "loss": -0.0469, "num_tokens": 11556048.0, "reward": -0.015625, "reward_std": 0.31534069776535034, "rewards/rollout_reward_func/mean": -0.015625, "rewards/rollout_reward_func/std": 0.3312823176383972, "sampling/importance_sampling_ratio/max": 1.6342065334320068, "sampling/importance_sampling_ratio/mean": 0.5772780179977417, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.57011032104492, "sampling/sampling_logp_difference/mean": 1.4640886783599854, "step": 258, "step_time": 18.653622422993067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0021428571199066937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021428571199066937, "completions/clipped_ratio": 0.25, "completions/max_length": 42.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 20.875, "completions/mean_terminated_length": 19.666667938232422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1580130346119404, "epoch": 0.00518, "frac_reward_zero_std": 0.0, "grad_norm": 0.2562621831893921, "kl": 0.31582158571109176, "learning_rate": 8.835854103996683e-06, "loss": -0.1156, "num_tokens": 11601335.0, "reward": 0.02187500335276127, "reward_std": 0.3563532829284668, "rewards/rollout_reward_func/mean": 0.02187500335276127, "rewards/rollout_reward_func/std": 0.3589719831943512, "sampling/importance_sampling_ratio/max": 1.6102739572525024, "sampling/importance_sampling_ratio/mean": 0.4332994222640991, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.1232795715332, "sampling/sampling_logp_difference/mean": 1.8580610752105713, "step": 259, "step_time": 19.136629674976575 }, { "clip_ratio/high_max": 0.008019943023100495, "clip_ratio/high_mean": 0.005087557598017156, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005087557598017156, "completions/clipped_ratio": 0.1875, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 20.03125, "completions/mean_terminated_length": 19.730770111083984, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9297917895019054, "epoch": 0.0052, "frac_reward_zero_std": 0.0, "grad_norm": 0.4093581438064575, "kl": 0.19154189340770245, "learning_rate": 8.825973366554254e-06, "loss": -0.1641, "num_tokens": 11647688.0, "reward": -0.04062500223517418, "reward_std": 0.31520992517471313, "rewards/rollout_reward_func/mean": -0.04062500223517418, "rewards/rollout_reward_func/std": 0.30570846796035767, "sampling/importance_sampling_ratio/max": 1.6478626728057861, "sampling/importance_sampling_ratio/mean": 0.5027758479118347, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.613243103027344, "sampling/sampling_logp_difference/mean": 1.5253673791885376, "step": 260, "step_time": 19.85501563399157 }, { "clip_ratio/high_max": 0.006830943748354912, "clip_ratio/high_mean": 0.003415471874177456, "clip_ratio/low_mean": 0.0007621950935572386, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0041776669677346945, "completions/clipped_ratio": 0.28125, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 20.71875, "completions/mean_terminated_length": 18.565217971801758, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0790954995900393, "epoch": 0.00522, "frac_reward_zero_std": 0.0, "grad_norm": 0.2851290702819824, "kl": 0.24549343716353178, "learning_rate": 8.816058650281758e-06, "loss": -0.1633, "num_tokens": 11692832.0, "reward": -0.04062500223517418, "reward_std": 0.30977678298950195, "rewards/rollout_reward_func/mean": -0.04062500223517418, "rewards/rollout_reward_func/std": 0.31608426570892334, "sampling/importance_sampling_ratio/max": 1.6067554950714111, "sampling/importance_sampling_ratio/mean": 0.408433198928833, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.988243103027344, "sampling/sampling_logp_difference/mean": 1.975189447402954, "step": 261, "step_time": 19.452777576996596 }, { "clip_ratio/high_max": 0.007291666930541396, "clip_ratio/high_mean": 0.003645833465270698, "clip_ratio/low_mean": 0.0025161030935123563, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006161936558783054, "completions/clipped_ratio": 0.15625, "completions/max_length": 34.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 17.65625, "completions/mean_terminated_length": 17.370370864868164, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.8937117196619511, "epoch": 0.00524, "frac_reward_zero_std": 0.0, "grad_norm": 0.43494492769241333, "kl": 0.28250861214473844, "learning_rate": 8.806110085960999e-06, "loss": -0.072, "num_tokens": 11737274.0, "reward": 0.02500000409781933, "reward_std": 0.28818202018737793, "rewards/rollout_reward_func/mean": 0.02500000409781933, "rewards/rollout_reward_func/std": 0.29729968309402466, "sampling/importance_sampling_ratio/max": 1.5935062170028687, "sampling/importance_sampling_ratio/mean": 0.665887713432312, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.87258529663086, "sampling/sampling_logp_difference/mean": 1.5518711805343628, "step": 262, "step_time": 20.31091477003065 }, { "clip_ratio/high_max": 0.011206974741071463, "clip_ratio/high_mean": 0.005603487370535731, "clip_ratio/low_mean": 0.0014880952658131719, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007091582519933581, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 18.59375, "completions/mean_terminated_length": 17.423076629638672, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0215714313089848, "epoch": 0.00526, "frac_reward_zero_std": 0.0, "grad_norm": 0.43815919756889343, "kl": 0.1381331468001008, "learning_rate": 8.79612780482027e-06, "loss": -0.0552, "num_tokens": 11782187.0, "reward": 0.05312500149011612, "reward_std": 0.2581050395965576, "rewards/rollout_reward_func/mean": 0.05312500149011612, "rewards/rollout_reward_func/std": 0.2651650309562683, "sampling/importance_sampling_ratio/max": 1.501410722732544, "sampling/importance_sampling_ratio/mean": 0.4908931255340576, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 34.15677261352539, "sampling/sampling_logp_difference/mean": 1.9318640232086182, "step": 263, "step_time": 18.507252863011672 }, { "clip_ratio/high_max": 0.006138392956927419, "clip_ratio/high_mean": 0.0030691964784637094, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030691964784637094, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.09375, "completions/mean_terminated_length": 15.653846740722656, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9798974376171827, "epoch": 0.00528, "frac_reward_zero_std": 0.0, "grad_norm": 0.3107761740684509, "kl": 0.24547715240623802, "learning_rate": 8.786111938532606e-06, "loss": -0.1128, "num_tokens": 11825410.0, "reward": 0.040625009685754776, "reward_std": 0.3262353241443634, "rewards/rollout_reward_func/mean": 0.040625009685754776, "rewards/rollout_reward_func/std": 0.313007652759552, "sampling/importance_sampling_ratio/max": 1.5431486368179321, "sampling/importance_sampling_ratio/mean": 0.5613915920257568, "sampling/importance_sampling_ratio/min": 7.006492321624085e-45, "sampling/sampling_logp_difference/max": 32.747589111328125, "sampling/sampling_logp_difference/mean": 1.9139723777770996, "step": 264, "step_time": 20.073809195047943 }, { "clip_ratio/high_max": 0.011602719314396381, "clip_ratio/high_mean": 0.005801359657198191, "clip_ratio/low_mean": 0.001869419647846371, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007670779305044562, "completions/clipped_ratio": 0.25, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 22.21875, "completions/mean_terminated_length": 21.5, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0434902030974627, "epoch": 0.0053, "frac_reward_zero_std": 0.0, "grad_norm": 0.34554538130760193, "kl": 0.10176776663865894, "learning_rate": 8.776062619214053e-06, "loss": -0.1657, "num_tokens": 11870007.0, "reward": 0.02500000409781933, "reward_std": 0.36952799558639526, "rewards/rollout_reward_func/mean": 0.02500000409781933, "rewards/rollout_reward_func/std": 0.3610021770000458, "sampling/importance_sampling_ratio/max": 1.291197657585144, "sampling/importance_sampling_ratio/mean": 0.38967210054397583, "sampling/importance_sampling_ratio/min": 3.6433760072445244e-44, "sampling/sampling_logp_difference/max": 33.7639045715332, "sampling/sampling_logp_difference/mean": 1.8162031173706055, "step": 265, "step_time": 19.68005008494947 }, { "clip_ratio/high_max": 0.013280083425343037, "clip_ratio/high_mean": 0.006640041712671518, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006640041712671518, "completions/clipped_ratio": 0.21875, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 18.59375, "completions/mean_terminated_length": 16.959999084472656, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9766780529171228, "epoch": 0.00532, "frac_reward_zero_std": 0.0, "grad_norm": 0.2872712314128876, "kl": 0.2082951725460589, "learning_rate": 8.765979979421928e-06, "loss": -0.0499, "num_tokens": 11915086.0, "reward": 9.313225746154785e-10, "reward_std": 0.33939996361732483, "rewards/rollout_reward_func/mean": 9.313225746154785e-10, "rewards/rollout_reward_func/std": 0.333118200302124, "sampling/importance_sampling_ratio/max": 1.5432604551315308, "sampling/importance_sampling_ratio/mean": 0.5474318861961365, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.56399154663086, "sampling/sampling_logp_difference/mean": 1.8388530015945435, "step": 266, "step_time": 18.860671415983234 }, { "clip_ratio/high_max": 0.005908613558858633, "clip_ratio/high_mean": 0.0029543067794293165, "clip_ratio/low_mean": 0.0032430581049993634, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00619736488442868, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 20.46875, "completions/mean_terminated_length": 19.55555534362793, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.071076463907957, "epoch": 0.00534, "frac_reward_zero_std": 0.0, "grad_norm": 0.4255862236022949, "kl": 0.23441203171387315, "learning_rate": 8.755864152153064e-06, "loss": -0.2232, "num_tokens": 11960509.0, "reward": -0.00937500037252903, "reward_std": 0.3210117220878601, "rewards/rollout_reward_func/mean": -0.00937500037252903, "rewards/rollout_reward_func/std": 0.32662433385849, "sampling/importance_sampling_ratio/max": 1.8411496877670288, "sampling/importance_sampling_ratio/mean": 0.47585296630859375, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.2014045715332, "sampling/sampling_logp_difference/mean": 2.154057025909424, "step": 267, "step_time": 20.554867532962817 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010775862028822303, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 20.46875, "completions/mean_terminated_length": 18.19999885559082, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9631120320409536, "epoch": 0.00536, "frac_reward_zero_std": 0.0, "grad_norm": 0.4019286036491394, "kl": 0.11019871279131621, "learning_rate": 8.745715270842064e-06, "loss": -0.1248, "num_tokens": 12005438.0, "reward": -0.06874999403953552, "reward_std": 0.33149006962776184, "rewards/rollout_reward_func/mean": -0.06874999403953552, "rewards/rollout_reward_func/std": 0.3550897240638733, "sampling/importance_sampling_ratio/max": 2.006124496459961, "sampling/importance_sampling_ratio/mean": 0.593513011932373, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.46194839477539, "sampling/sampling_logp_difference/mean": 1.6727427244186401, "step": 268, "step_time": 19.313792355998885 }, { "clip_ratio/high_max": 0.006481481716036797, "clip_ratio/high_mean": 0.0032407408580183983, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032407408580183983, "completions/clipped_ratio": 0.1875, "completions/max_length": 64.0, "completions/max_terminated_length": 64.0, "completions/mean_length": 20.4375, "completions/mean_terminated_length": 19.76923179626465, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.064407417550683, "epoch": 0.00538, "frac_reward_zero_std": 0.0, "grad_norm": 0.2839448153972626, "kl": 0.23975114966742694, "learning_rate": 8.73553346935953e-06, "loss": -0.044, "num_tokens": 12050830.0, "reward": 0.04062500223517418, "reward_std": 0.331950306892395, "rewards/rollout_reward_func/mean": 0.04062500223517418, "rewards/rollout_reward_func/std": 0.3444110155105591, "sampling/importance_sampling_ratio/max": 1.5253820419311523, "sampling/importance_sampling_ratio/mean": 0.438607394695282, "sampling/importance_sampling_ratio/min": 1.754425677334671e-42, "sampling/sampling_logp_difference/max": 33.388092041015625, "sampling/sampling_logp_difference/mean": 1.8537006378173828, "step": 269, "step_time": 20.879470360960113 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.004303774971049279, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006256899971049279, "completions/clipped_ratio": 0.3125, "completions/max_length": 47.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 21.09375, "completions/mean_terminated_length": 18.090909957885742, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.083676552399993, "epoch": 0.0054, "frac_reward_zero_std": 0.0, "grad_norm": 0.3095130920410156, "kl": 0.21731114946305752, "learning_rate": 8.725318882010307e-06, "loss": -0.1918, "num_tokens": 12095918.0, "reward": -0.11562500149011612, "reward_std": 0.2938721776008606, "rewards/rollout_reward_func/mean": -0.11562500149011612, "rewards/rollout_reward_func/std": 0.2974183261394501, "sampling/importance_sampling_ratio/max": 1.5685412883758545, "sampling/importance_sampling_ratio/mean": 0.4322134852409363, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.663848876953125, "sampling/sampling_logp_difference/mean": 2.0341224670410156, "step": 270, "step_time": 19.419713412964484 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024999999441206455, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 19.84375, "completions/mean_terminated_length": 16.363636016845703, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0365325585007668, "epoch": 0.00542, "frac_reward_zero_std": 0.0, "grad_norm": 0.2623957395553589, "kl": 0.11958370136562735, "learning_rate": 8.715071643531708e-06, "loss": -0.1036, "num_tokens": 12141599.0, "reward": -0.028124995529651642, "reward_std": 0.3072796165943146, "rewards/rollout_reward_func/mean": -0.028124995529651642, "rewards/rollout_reward_func/std": 0.3061038851737976, "sampling/importance_sampling_ratio/max": 1.473970651626587, "sampling/importance_sampling_ratio/mean": 0.47593969106674194, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.86323928833008, "sampling/sampling_logp_difference/mean": 2.101132392883301, "step": 271, "step_time": 18.72931693699502 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0019602272659540176, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031176346819847822, "completions/clipped_ratio": 0.34375, "completions/max_length": 48.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 23.1875, "completions/mean_terminated_length": 20.285715103149414, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.0983199961483479, "epoch": 0.00544, "frac_reward_zero_std": 0.0, "grad_norm": 0.2700115740299225, "kl": 0.16539981844834983, "learning_rate": 8.704791889091737e-06, "loss": -0.1369, "num_tokens": 12187357.0, "reward": -0.12812499701976776, "reward_std": 0.3322683572769165, "rewards/rollout_reward_func/mean": -0.12812499701976776, "rewards/rollout_reward_func/std": 0.36653071641921997, "sampling/importance_sampling_ratio/max": 1.47135329246521, "sampling/importance_sampling_ratio/mean": 0.34029221534729004, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.538848876953125, "sampling/sampling_logp_difference/mean": 1.896970510482788, "step": 272, "step_time": 21.266726856993046 }, { "clip_ratio/high_max": 0.010139375925064087, "clip_ratio/high_mean": 0.0050696879625320435, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006714424816891551, "completions/clipped_ratio": 0.375, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 19.59375, "completions/mean_terminated_length": 16.700000762939453, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1769500505179167, "epoch": 0.00546, "frac_reward_zero_std": 0.0, "grad_norm": 0.5324320793151855, "kl": 0.26532530318945646, "learning_rate": 8.6944797542873e-06, "loss": -0.0296, "num_tokens": 12232533.0, "reward": 0.006250001490116119, "reward_std": 0.32206785678863525, "rewards/rollout_reward_func/mean": 0.006250001490116119, "rewards/rollout_reward_func/std": 0.32122498750686646, "sampling/importance_sampling_ratio/max": 2.320683717727661, "sampling/importance_sampling_ratio/mean": 0.5124391317367554, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.58694839477539, "sampling/sampling_logp_difference/mean": 1.8990964889526367, "step": 273, "step_time": 19.469956133994856 }, { "clip_ratio/high_max": 0.004903846187517047, "clip_ratio/high_mean": 0.003529509180225432, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003529509180225432, "completions/clipped_ratio": 0.375, "completions/max_length": 46.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 20.84375, "completions/mean_terminated_length": 17.75, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.031591897830367, "epoch": 0.00548, "frac_reward_zero_std": 0.0, "grad_norm": 0.28639864921569824, "kl": 0.2167814418207854, "learning_rate": 8.68413537514243e-06, "loss": -0.1968, "num_tokens": 12276966.0, "reward": -0.03750000149011612, "reward_std": 0.3600400388240814, "rewards/rollout_reward_func/mean": -0.03750000149011612, "rewards/rollout_reward_func/std": 0.3526398241519928, "sampling/importance_sampling_ratio/max": 1.5873879194259644, "sampling/importance_sampling_ratio/mean": 0.48103630542755127, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.70139694213867, "sampling/sampling_logp_difference/mean": 1.938406229019165, "step": 274, "step_time": 20.13515230100893 }, { "clip_ratio/high_max": 0.01040047942660749, "clip_ratio/high_mean": 0.005200239713303745, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005200239713303745, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.125, "completions/mean_terminated_length": 17.03999900817871, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9256341923028231, "epoch": 0.0055, "frac_reward_zero_std": 0.0, "grad_norm": 0.4650212228298187, "kl": 0.12914481409825385, "learning_rate": 8.673758888106481e-06, "loss": 0.0648, "num_tokens": 12321456.0, "reward": 0.06562499701976776, "reward_std": 0.21824076771736145, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.2103903889656067, "sampling/importance_sampling_ratio/max": 1.7436193227767944, "sampling/importance_sampling_ratio/mean": 0.576916515827179, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.79195022583008, "sampling/sampling_logp_difference/mean": 1.7267452478408813, "step": 275, "step_time": 18.905274645978352 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0006944444612599909, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018518518772907555, "completions/clipped_ratio": 0.3125, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 20.78125, "completions/mean_terminated_length": 21.636363983154297, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9144557900726795, "epoch": 0.00552, "frac_reward_zero_std": 0.0, "grad_norm": 0.42588484287261963, "kl": 0.2750626665074378, "learning_rate": 8.663350430052328e-06, "loss": -0.1474, "num_tokens": 12366837.0, "reward": -0.03124999813735485, "reward_std": 0.28983697295188904, "rewards/rollout_reward_func/mean": -0.03124999813735485, "rewards/rollout_reward_func/std": 0.2867363691329956, "sampling/importance_sampling_ratio/max": 1.513139009475708, "sampling/importance_sampling_ratio/mean": 0.5168040990829468, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.726356506347656, "sampling/sampling_logp_difference/mean": 1.5086976289749146, "step": 276, "step_time": 19.57238590097404 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028467909432947636, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.875, "completions/mean_terminated_length": 19.08333396911621, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.8898480888456106, "epoch": 0.00554, "frac_reward_zero_std": 0.0, "grad_norm": 0.28554487228393555, "kl": 0.16259774519130588, "learning_rate": 8.652910138274573e-06, "loss": -0.1906, "num_tokens": 12410093.0, "reward": 0.009375002235174179, "reward_std": 0.3829250931739807, "rewards/rollout_reward_func/mean": 0.009375002235174179, "rewards/rollout_reward_func/std": 0.36839616298675537, "sampling/importance_sampling_ratio/max": 1.4458707571029663, "sampling/importance_sampling_ratio/mean": 0.5289548635482788, "sampling/importance_sampling_ratio/min": 5.633219826585765e-43, "sampling/sampling_logp_difference/max": 33.446327209472656, "sampling/sampling_logp_difference/mean": 1.5238616466522217, "step": 277, "step_time": 19.51712030201452 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.0031414691475220025, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004629564413335174, "completions/clipped_ratio": 0.15625, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 21.125, "completions/mean_terminated_length": 20.259260177612305, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.223867291584611, "epoch": 0.00556, "frac_reward_zero_std": 0.0, "grad_norm": 0.22413474321365356, "kl": 0.2881440273486078, "learning_rate": 8.642438150487718e-06, "loss": -0.1579, "num_tokens": 12454546.0, "reward": 0.04062499850988388, "reward_std": 0.32171469926834106, "rewards/rollout_reward_func/mean": 0.04062499850988388, "rewards/rollout_reward_func/std": 0.3290841281414032, "sampling/importance_sampling_ratio/max": 1.7056059837341309, "sampling/importance_sampling_ratio/mean": 0.3052476644515991, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.15774154663086, "sampling/sampling_logp_difference/mean": 2.3658251762390137, "step": 278, "step_time": 19.50308139399567 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.002731092448811978, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00408978812629357, "completions/clipped_ratio": 0.25, "completions/max_length": 35.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 19.1875, "completions/mean_terminated_length": 16.916667938232422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0108796674758196, "epoch": 0.00558, "frac_reward_zero_std": 0.0, "grad_norm": 0.4275679290294647, "kl": 0.2710108351893723, "learning_rate": 8.631934604824366e-06, "loss": -0.0786, "num_tokens": 12499363.0, "reward": 0.009375002235174179, "reward_std": 0.29579541087150574, "rewards/rollout_reward_func/mean": 0.009375002235174179, "rewards/rollout_reward_func/std": 0.31249192357063293, "sampling/importance_sampling_ratio/max": 1.5504387617111206, "sampling/importance_sampling_ratio/mean": 0.5294078588485718, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.75492477416992, "sampling/sampling_logp_difference/mean": 1.943395972251892, "step": 279, "step_time": 19.17744706600206 }, { "clip_ratio/high_max": 0.014295917470008135, "clip_ratio/high_mean": 0.007147958735004067, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007147958735004067, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.21875, "completions/mean_terminated_length": 18.807693481445312, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.1976877860724926, "epoch": 0.0056, "frac_reward_zero_std": 0.0, "grad_norm": 0.2625780999660492, "kl": 0.24334806995466352, "learning_rate": 8.62139963983338e-06, "loss": -0.0033, "num_tokens": 12545777.0, "reward": -0.14375001192092896, "reward_std": 0.3270583748817444, "rewards/rollout_reward_func/mean": -0.14375001192092896, "rewards/rollout_reward_func/std": 0.32122495770454407, "sampling/importance_sampling_ratio/max": 2.054732322692871, "sampling/importance_sampling_ratio/mean": 0.4465973973274231, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.2326545715332, "sampling/sampling_logp_difference/mean": 1.9490959644317627, "step": 280, "step_time": 19.689771221994306 }, { "clip_ratio/high_max": 0.008506865007802844, "clip_ratio/high_mean": 0.004253432503901422, "clip_ratio/low_mean": 0.002436281880363822, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006689714384265244, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 21.21875, "completions/mean_terminated_length": 20.904762268066406, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1104110069572926, "epoch": 0.00562, "frac_reward_zero_std": 0.0, "grad_norm": 0.29500752687454224, "kl": 0.1708815669408068, "learning_rate": 8.610833394478073e-06, "loss": -0.1092, "num_tokens": 12591782.0, "reward": -0.04374999925494194, "reward_std": 0.32421866059303284, "rewards/rollout_reward_func/mean": -0.04374999925494194, "rewards/rollout_reward_func/std": 0.3181980550289154, "sampling/importance_sampling_ratio/max": 1.3897576332092285, "sampling/importance_sampling_ratio/mean": 0.3308930993080139, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.019474029541016, "sampling/sampling_logp_difference/mean": 2.0535552501678467, "step": 281, "step_time": 19.492985308010248 }, { "clip_ratio/high_max": 0.006761695956811309, "clip_ratio/high_mean": 0.0033808479784056544, "clip_ratio/low_mean": 0.0010416667209938169, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004422514699399471, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.65625, "completions/mean_terminated_length": 15.739130973815918, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.996479470282793, "epoch": 0.00564, "frac_reward_zero_std": 0.0, "grad_norm": 0.37768417596817017, "kl": 0.3443316627526656, "learning_rate": 8.60023600813436e-06, "loss": -0.1482, "num_tokens": 12636873.0, "reward": -0.07187499850988388, "reward_std": 0.3446335196495056, "rewards/rollout_reward_func/mean": -0.07187499850988388, "rewards/rollout_reward_func/std": 0.3475577235221863, "sampling/importance_sampling_ratio/max": 1.5122729539871216, "sampling/importance_sampling_ratio/mean": 0.48110252618789673, "sampling/importance_sampling_ratio/min": 2.2420775429197073e-44, "sampling/sampling_logp_difference/max": 32.73259735107422, "sampling/sampling_logp_difference/mean": 1.7347538471221924, "step": 282, "step_time": 20.04372068597877 }, { "clip_ratio/high_max": 0.0022321429569274187, "clip_ratio/high_mean": 0.0011160714784637094, "clip_ratio/low_mean": 0.0007102272938936949, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018262987723574042, "completions/clipped_ratio": 0.28125, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 19.0625, "completions/mean_terminated_length": 18.478261947631836, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.245432786643505, "epoch": 0.00566, "frac_reward_zero_std": 0.0, "grad_norm": 0.5053436756134033, "kl": 0.28530652180779725, "learning_rate": 8.589607620588937e-06, "loss": -0.0347, "num_tokens": 12682402.0, "reward": -0.03750000149011612, "reward_std": 0.23585055768489838, "rewards/rollout_reward_func/mean": -0.03750000149011612, "rewards/rollout_reward_func/std": 0.25495100021362305, "sampling/importance_sampling_ratio/max": 2.3820900917053223, "sampling/importance_sampling_ratio/mean": 0.40024158358573914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.45698928833008, "sampling/sampling_logp_difference/mean": 2.2035880088806152, "step": 283, "step_time": 19.202592264991836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.008333333651535213, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008333333651535213, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.46875, "completions/mean_terminated_length": 15.34782600402832, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0800829455256462, "epoch": 0.00568, "frac_reward_zero_std": 0.0, "grad_norm": 0.4317224323749542, "kl": 0.1638176254928112, "learning_rate": 8.578948372037416e-06, "loss": -0.038, "num_tokens": 12726802.0, "reward": 0.05312499776482582, "reward_std": 0.3002026379108429, "rewards/rollout_reward_func/mean": 0.05312499776482582, "rewards/rollout_reward_func/std": 0.314164936542511, "sampling/importance_sampling_ratio/max": 1.5565040111541748, "sampling/importance_sampling_ratio/mean": 0.5339411497116089, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.10988235473633, "sampling/sampling_logp_difference/mean": 1.7719718217849731, "step": 284, "step_time": 19.305352762006805 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0007102272938936949, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020689229713752866, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 21.8125, "completions/mean_terminated_length": 19.461538314819336, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.006107896566391, "epoch": 0.0057, "frac_reward_zero_std": 0.0, "grad_norm": 0.27813175320625305, "kl": 0.2564269485883415, "learning_rate": 8.568258403082492e-06, "loss": -0.0506, "num_tokens": 12772765.0, "reward": -0.125, "reward_std": 0.3012442886829376, "rewards/rollout_reward_func/mean": -0.125, "rewards/rollout_reward_func/std": 0.3005371689796448, "sampling/importance_sampling_ratio/max": 1.4026850461959839, "sampling/importance_sampling_ratio/mean": 0.3625587224960327, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.51948547363281, "sampling/sampling_logp_difference/mean": 1.947901964187622, "step": 285, "step_time": 19.837843164015794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0017361111240461469, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017361111240461469, "completions/clipped_ratio": 0.1875, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 19.125, "completions/mean_terminated_length": 17.38461685180664, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9661659859120846, "epoch": 0.00572, "frac_reward_zero_std": 0.0, "grad_norm": 0.35879626870155334, "kl": 0.148744429461658, "learning_rate": 8.557537854732082e-06, "loss": -0.1363, "num_tokens": 12816447.0, "reward": 0.06562500447034836, "reward_std": 0.293767511844635, "rewards/rollout_reward_func/mean": 0.06562500447034836, "rewards/rollout_reward_func/std": 0.29028281569480896, "sampling/importance_sampling_ratio/max": 1.8539462089538574, "sampling/importance_sampling_ratio/mean": 0.5517975687980652, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.2425537109375, "sampling/sampling_logp_difference/mean": 1.8955683708190918, "step": 286, "step_time": 19.241275946958922 }, { "clip_ratio/high_max": 0.006481481716036797, "clip_ratio/high_mean": 0.0032407408580183983, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004490740830078721, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.5, "completions/mean_terminated_length": 16.65517234802246, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8895301632583141, "epoch": 0.00574, "frac_reward_zero_std": 0.0, "grad_norm": 0.41344717144966125, "kl": 0.20713496417738497, "learning_rate": 8.546786868397465e-06, "loss": -0.0516, "num_tokens": 12859811.0, "reward": 0.05624999850988388, "reward_std": 0.28392916917800903, "rewards/rollout_reward_func/mean": 0.05624999850988388, "rewards/rollout_reward_func/std": 0.31308814883232117, "sampling/importance_sampling_ratio/max": 1.7056388854980469, "sampling/importance_sampling_ratio/mean": 0.6784973740577698, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.65774154663086, "sampling/sampling_logp_difference/mean": 1.5109727382659912, "step": 287, "step_time": 19.16022094305663 }, { "clip_ratio/high_max": 0.003289473708719015, "clip_ratio/high_mean": 0.0016447368543595076, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028947368264198303, "completions/clipped_ratio": 0.21875, "completions/max_length": 35.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 19.9375, "completions/mean_terminated_length": 18.959999084472656, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.141565041616559, "epoch": 0.00576, "frac_reward_zero_std": 0.0, "grad_norm": 0.30633482336997986, "kl": 0.2570801127003506, "learning_rate": 8.536005585891422e-06, "loss": -0.0986, "num_tokens": 12905296.0, "reward": -0.10000000149011612, "reward_std": 0.35442930459976196, "rewards/rollout_reward_func/mean": -0.10000000149011612, "rewards/rollout_reward_func/std": 0.35560035705566406, "sampling/importance_sampling_ratio/max": 1.39195716381073, "sampling/importance_sampling_ratio/mean": 0.4338153004646301, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.34297180175781, "sampling/sampling_logp_difference/mean": 2.1598312854766846, "step": 288, "step_time": 18.93287036602851 }, { "clip_ratio/high_max": 0.005434782709926367, "clip_ratio/high_mean": 0.0027173913549631834, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0027173913549631834, "completions/clipped_ratio": 0.1875, "completions/max_length": 64.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.28125, "completions/mean_terminated_length": 16.576923370361328, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.9415320158004761, "epoch": 0.00578, "frac_reward_zero_std": 0.0, "grad_norm": 0.36727169156074524, "kl": 0.13895062683150172, "learning_rate": 8.525194149426354e-06, "loss": -0.1154, "num_tokens": 12950513.0, "reward": -0.05625000223517418, "reward_std": 0.3377103805541992, "rewards/rollout_reward_func/mean": -0.05625000223517418, "rewards/rollout_reward_func/std": 0.3537243902683258, "sampling/importance_sampling_ratio/max": 1.380019187927246, "sampling/importance_sampling_ratio/mean": 0.501772940158844, "sampling/importance_sampling_ratio/min": 4.568232993698904e-43, "sampling/sampling_logp_difference/max": 32.9300537109375, "sampling/sampling_logp_difference/mean": 1.9468934535980225, "step": 289, "step_time": 20.147858614000143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.59375, "completions/mean_terminated_length": 17.25, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0248133894056082, "epoch": 0.0058, "frac_reward_zero_std": 0.0, "grad_norm": 0.3852458894252777, "kl": 0.11664655874483287, "learning_rate": 8.514352701612418e-06, "loss": -0.0475, "num_tokens": 12995334.0, "reward": 3.725290298461914e-09, "reward_std": 0.25891220569610596, "rewards/rollout_reward_func/mean": 3.725290298461914e-09, "rewards/rollout_reward_func/std": 0.27591490745544434, "sampling/importance_sampling_ratio/max": 1.9170994758605957, "sampling/importance_sampling_ratio/mean": 0.603030800819397, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 32.966331481933594, "sampling/sampling_logp_difference/mean": 1.7900623083114624, "step": 290, "step_time": 18.704214771976694 }, { "clip_ratio/high_max": 0.011479302076622844, "clip_ratio/high_mean": 0.005739651038311422, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0068970584543421865, "completions/clipped_ratio": 0.28125, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 16.90625, "completions/mean_terminated_length": 16.2608699798584, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9756611157208681, "epoch": 0.00582, "frac_reward_zero_std": 0.0, "grad_norm": 0.3960903584957123, "kl": 0.24561715475283563, "learning_rate": 8.503481385455637e-06, "loss": -0.1999, "num_tokens": 13038897.0, "reward": 0.02500000223517418, "reward_std": 0.3021829426288605, "rewards/rollout_reward_func/mean": 0.02500000223517418, "rewards/rollout_reward_func/std": 0.30900463461875916, "sampling/importance_sampling_ratio/max": 1.6861052513122559, "sampling/importance_sampling_ratio/mean": 0.6460039019584656, "sampling/importance_sampling_ratio/min": 5.605193857299268e-45, "sampling/sampling_logp_difference/max": 33.5075798034668, "sampling/sampling_logp_difference/mean": 1.7552847862243652, "step": 291, "step_time": 18.837703270997736 }, { "clip_ratio/high_max": 0.010392366209998727, "clip_ratio/high_mean": 0.005196183104999363, "clip_ratio/low_mean": 0.0025161030935123563, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00771228619851172, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 19.0625, "completions/mean_terminated_length": 16.95652198791504, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1023801919072866, "epoch": 0.00584, "frac_reward_zero_std": 0.0, "grad_norm": 0.2850721478462219, "kl": 0.48658510053064674, "learning_rate": 8.492580344356023e-06, "loss": 0.0091, "num_tokens": 13084349.0, "reward": -0.006250002421438694, "reward_std": 0.2744818925857544, "rewards/rollout_reward_func/mean": -0.006250002421438694, "rewards/rollout_reward_func/std": 0.2793281376361847, "sampling/importance_sampling_ratio/max": 1.396643877029419, "sampling/importance_sampling_ratio/mean": 0.37507230043411255, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.82961654663086, "sampling/sampling_logp_difference/mean": 2.184288501739502, "step": 292, "step_time": 19.15769376498065 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.005058241076767445, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006260164198465645, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 21.21875, "completions/mean_terminated_length": 19.791667938232422, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.2965414337813854, "epoch": 0.00586, "frac_reward_zero_std": 0.0, "grad_norm": 0.40925729274749756, "kl": 0.11496778624132276, "learning_rate": 8.481649722105677e-06, "loss": -0.0627, "num_tokens": 13130866.0, "reward": -0.02187499776482582, "reward_std": 0.2990390956401825, "rewards/rollout_reward_func/mean": -0.02187499776482582, "rewards/rollout_reward_func/std": 0.32001447677612305, "sampling/importance_sampling_ratio/max": 1.6899960041046143, "sampling/importance_sampling_ratio/mean": 0.3917546570301056, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.4826545715332, "sampling/sampling_logp_difference/mean": 2.326326847076416, "step": 293, "step_time": 20.08784240503155 }, { "clip_ratio/high_max": 0.015059125958941877, "clip_ratio/high_mean": 0.0075295629794709384, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009761705936398357, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 47.0, "completions/mean_length": 23.21875, "completions/mean_terminated_length": 21.479999542236328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2986610494554043, "epoch": 0.00588, "frac_reward_zero_std": 0.0, "grad_norm": 0.21920351684093475, "kl": 0.1513478176202625, "learning_rate": 8.470689662886895e-06, "loss": -0.0289, "num_tokens": 13177289.0, "reward": -0.10000000149011612, "reward_std": 0.34915691614151, "rewards/rollout_reward_func/mean": -0.10000000149011612, "rewards/rollout_reward_func/std": 0.34077709913253784, "sampling/importance_sampling_ratio/max": 1.501708984375, "sampling/importance_sampling_ratio/mean": 0.3669443428516388, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.04514694213867, "sampling/sampling_logp_difference/mean": 2.451019287109375, "step": 294, "step_time": 21.106651607973618 }, { "clip_ratio/high_max": 0.0031250000465661287, "clip_ratio/high_mean": 0.0015625000232830644, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002812499995343387, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 18.09375, "completions/mean_terminated_length": 16.22727394104004, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.8636256493628025, "epoch": 0.0059, "frac_reward_zero_std": 0.0, "grad_norm": 0.45120319724082947, "kl": 0.11360600567422807, "learning_rate": 8.459700311270268e-06, "loss": -0.1831, "num_tokens": 13220991.0, "reward": -1.862645149230957e-09, "reward_std": 0.3087533414363861, "rewards/rollout_reward_func/mean": -1.862645149230957e-09, "rewards/rollout_reward_func/std": 0.313152551651001, "sampling/importance_sampling_ratio/max": 1.7640376091003418, "sampling/importance_sampling_ratio/mean": 0.6877671480178833, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.24319076538086, "sampling/sampling_logp_difference/mean": 1.5135242938995361, "step": 295, "step_time": 18.56463732899283 }, { "clip_ratio/high_max": 0.004670516354963183, "clip_ratio/high_mean": 0.0023352581774815917, "clip_ratio/low_mean": 0.0027225379599258304, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005057796137407422, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 21.0625, "completions/mean_terminated_length": 19.399999618530273, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.1927586160600185, "epoch": 0.00592, "frac_reward_zero_std": 0.0, "grad_norm": 0.3326604664325714, "kl": 0.19867980666458607, "learning_rate": 8.44868181221277e-06, "loss": -0.1857, "num_tokens": 13265833.0, "reward": -0.0624999962747097, "reward_std": 0.31619593501091003, "rewards/rollout_reward_func/mean": -0.0624999962747097, "rewards/rollout_reward_func/std": 0.33093222975730896, "sampling/importance_sampling_ratio/max": 1.5853914022445679, "sampling/importance_sampling_ratio/mean": 0.4003182649612427, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.38890075683594, "sampling/sampling_logp_difference/mean": 2.0007851123809814, "step": 296, "step_time": 20.676964980986668 }, { "clip_ratio/high_max": 0.011869952781125903, "clip_ratio/high_mean": 0.0059349763905629516, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007184976362623274, "completions/clipped_ratio": 0.15625, "completions/max_length": 31.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.40625, "completions/mean_terminated_length": 18.703702926635742, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1858059540390968, "epoch": 0.00594, "frac_reward_zero_std": 0.0, "grad_norm": 0.2512955963611603, "kl": 0.25369319529272616, "learning_rate": 8.437634311055855e-06, "loss": -0.0197, "num_tokens": 13311655.0, "reward": -0.00937499850988388, "reward_std": 0.28523871302604675, "rewards/rollout_reward_func/mean": -0.00937499850988388, "rewards/rollout_reward_func/std": 0.2763347327709198, "sampling/importance_sampling_ratio/max": 1.4936150312423706, "sampling/importance_sampling_ratio/mean": 0.36057692766189575, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.19565963745117, "sampling/sampling_logp_difference/mean": 2.377462387084961, "step": 297, "step_time": 19.500819630993647 }, { "clip_ratio/high_max": 0.0021551724057644606, "clip_ratio/high_mean": 0.0010775862028822303, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010775862028822303, "completions/clipped_ratio": 0.28125, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 18.875, "completions/mean_terminated_length": 17.34782600402832, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8848795611411333, "epoch": 0.00596, "frac_reward_zero_std": 0.0, "grad_norm": 0.29283788800239563, "kl": 0.11738456878811121, "learning_rate": 8.42655795352353e-06, "loss": -0.1548, "num_tokens": 13355366.0, "reward": 0.02187500149011612, "reward_std": 0.3585911691188812, "rewards/rollout_reward_func/mean": 0.02187500149011612, "rewards/rollout_reward_func/std": 0.3571702241897583, "sampling/importance_sampling_ratio/max": 1.3841389417648315, "sampling/importance_sampling_ratio/mean": 0.5963135957717896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.826385498046875, "sampling/sampling_logp_difference/mean": 1.7560808658599854, "step": 298, "step_time": 18.969993231003173 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012499999720603228, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 19.0625, "completions/mean_terminated_length": 17.478261947631836, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.0900061316788197, "epoch": 0.00598, "frac_reward_zero_std": 0.0, "grad_norm": 0.5444231033325195, "kl": 0.09568556095473468, "learning_rate": 8.415452885720438e-06, "loss": -0.2155, "num_tokens": 13399472.0, "reward": 0.01875000074505806, "reward_std": 0.25013720989227295, "rewards/rollout_reward_func/mean": 0.01875000074505806, "rewards/rollout_reward_func/std": 0.27642592787742615, "sampling/importance_sampling_ratio/max": 1.5174751281738281, "sampling/importance_sampling_ratio/mean": 0.5554988384246826, "sampling/importance_sampling_ratio/min": 1.1210387714598537e-44, "sampling/sampling_logp_difference/max": 33.48265075683594, "sampling/sampling_logp_difference/mean": 1.8168352842330933, "step": 299, "step_time": 19.155810893003945 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.0035416666651144624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005277777789160609, "completions/clipped_ratio": 0.53125, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 20.96875, "completions/mean_terminated_length": 19.80000114440918, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1045855358242989, "epoch": 0.006, "frac_reward_zero_std": 0.0, "grad_norm": 0.37755709886550903, "kl": 0.11973665782716125, "learning_rate": 8.40431925412993e-06, "loss": -0.1407, "num_tokens": 13444461.0, "reward": -0.05000000447034836, "reward_std": 0.26166898012161255, "rewards/rollout_reward_func/mean": -0.05000000447034836, "rewards/rollout_reward_func/std": 0.29729968309402466, "sampling/importance_sampling_ratio/max": 1.707287311553955, "sampling/importance_sampling_ratio/mean": 0.5435250997543335, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.763885498046875, "sampling/sampling_logp_difference/mean": 1.842865228652954, "step": 300, "step_time": 19.272337614980643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0028332124929875135, "clip_ratio/low_min": 0.001179245300590992, "clip_ratio/region_mean": 0.0028332124929875135, "completions/clipped_ratio": 0.1875, "completions/max_length": 53.0, "completions/max_terminated_length": 53.0, "completions/mean_length": 21.21875, "completions/mean_terminated_length": 20.653846740722656, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.07236173376441, "epoch": 0.00602, "frac_reward_zero_std": 0.0, "grad_norm": 0.3635254204273224, "kl": 0.3251118811313063, "learning_rate": 8.393157205612137e-06, "loss": -0.2363, "num_tokens": 13490276.0, "reward": -0.09687499701976776, "reward_std": 0.3360159695148468, "rewards/rollout_reward_func/mean": -0.09687499701976776, "rewards/rollout_reward_func/std": 0.34123530983924866, "sampling/importance_sampling_ratio/max": 1.693783164024353, "sampling/importance_sampling_ratio/mean": 0.39707958698272705, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.75148010253906, "sampling/sampling_logp_difference/mean": 2.118870735168457, "step": 301, "step_time": 19.827088025966077 }, { "clip_ratio/high_max": 0.005681818351149559, "clip_ratio/high_mean": 0.0028409091755747795, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028409091755747795, "completions/clipped_ratio": 0.28125, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 19.9375, "completions/mean_terminated_length": 18.782609939575195, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0698132626712322, "epoch": 0.00604, "frac_reward_zero_std": 0.0, "grad_norm": 0.31085076928138733, "kl": 0.14765486167743802, "learning_rate": 8.38196688740202e-06, "loss": 0.011, "num_tokens": 13535984.0, "reward": 0.015625005587935448, "reward_std": 0.2963561415672302, "rewards/rollout_reward_func/mean": 0.015625005587935448, "rewards/rollout_reward_func/std": 0.29414650797843933, "sampling/importance_sampling_ratio/max": 1.3765878677368164, "sampling/importance_sampling_ratio/mean": 0.439159631729126, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.67014694213867, "sampling/sampling_logp_difference/mean": 1.8473989963531494, "step": 302, "step_time": 19.41825120896101 }, { "clip_ratio/high_max": 0.0059912854339927435, "clip_ratio/high_mean": 0.0029956427169963717, "clip_ratio/low_mean": 0.002864583395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005860226112417877, "completions/clipped_ratio": 0.3125, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.875, "completions/mean_terminated_length": 16.90909194946289, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0673960447311401, "epoch": 0.00606, "frac_reward_zero_std": 0.0, "grad_norm": 0.402617871761322, "kl": 0.21540673333220184, "learning_rate": 8.370748447107445e-06, "loss": -0.1477, "num_tokens": 13581824.0, "reward": -0.04062499850988388, "reward_std": 0.374575138092041, "rewards/rollout_reward_func/mean": -0.04062499850988388, "rewards/rollout_reward_func/std": 0.3766228258609772, "sampling/importance_sampling_ratio/max": 2.5539660453796387, "sampling/importance_sampling_ratio/mean": 0.5206034779548645, "sampling/importance_sampling_ratio/min": 4.203895392974451e-45, "sampling/sampling_logp_difference/max": 33.51945877075195, "sampling/sampling_logp_difference/mean": 1.811484456062317, "step": 303, "step_time": 19.42876817600336 }, { "clip_ratio/high_max": 0.004999999888241291, "clip_ratio/high_mean": 0.0024999999441206455, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024999999441206455, "completions/clipped_ratio": 0.3125, "completions/max_length": 48.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 20.90625, "completions/mean_terminated_length": 17.363636016845703, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.036175973713398, "epoch": 0.00608, "frac_reward_zero_std": 0.0, "grad_norm": 0.46306341886520386, "kl": 0.35197410359978676, "learning_rate": 8.359502032707219e-06, "loss": -0.0937, "num_tokens": 13627979.0, "reward": -0.02187499590218067, "reward_std": 0.3460027575492859, "rewards/rollout_reward_func/mean": -0.02187499590218067, "rewards/rollout_reward_func/std": 0.3580722510814667, "sampling/importance_sampling_ratio/max": 1.8582143783569336, "sampling/importance_sampling_ratio/mean": 0.47987595200538635, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.031768798828125, "sampling/sampling_logp_difference/mean": 1.9335596561431885, "step": 304, "step_time": 20.696591429019463 }, { "clip_ratio/high_max": 0.001953125, "clip_ratio/high_mean": 0.0009765625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009765625, "completions/clipped_ratio": 0.3125, "completions/max_length": 46.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.5625, "completions/mean_terminated_length": 14.545454978942871, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.8995589725673199, "epoch": 0.0061, "frac_reward_zero_std": 0.0, "grad_norm": 0.3352073132991791, "kl": 0.2570621606428176, "learning_rate": 8.34822779254915e-06, "loss": -0.0509, "num_tokens": 13672445.0, "reward": -0.04374999552965164, "reward_std": 0.27320587635040283, "rewards/rollout_reward_func/mean": -0.04374999552965164, "rewards/rollout_reward_func/std": 0.33498916029930115, "sampling/importance_sampling_ratio/max": 1.756321668624878, "sampling/importance_sampling_ratio/mean": 0.5228093862533569, "sampling/importance_sampling_ratio/min": 3.797518838320254e-43, "sampling/sampling_logp_difference/max": 33.375511169433594, "sampling/sampling_logp_difference/mean": 1.5996031761169434, "step": 305, "step_time": 19.021806657023262 }, { "clip_ratio/high_max": 0.005073052132502198, "clip_ratio/high_mean": 0.002536526066251099, "clip_ratio/low_mean": 0.002273478894494474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004810004960745573, "completions/clipped_ratio": 0.15625, "completions/max_length": 30.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 17.375, "completions/mean_terminated_length": 17.11111068725586, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9586461260914803, "epoch": 0.00612, "frac_reward_zero_std": 0.0, "grad_norm": 0.7339118719100952, "kl": 0.22559396922588348, "learning_rate": 8.336925875348093e-06, "loss": -0.1113, "num_tokens": 13716918.0, "reward": 0.06875000149011612, "reward_std": 0.24838952720165253, "rewards/rollout_reward_func/mean": 0.06875000149011612, "rewards/rollout_reward_func/std": 0.2867363691329956, "sampling/importance_sampling_ratio/max": 2.8900210857391357, "sampling/importance_sampling_ratio/mean": 0.5892590880393982, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.250511169433594, "sampling/sampling_logp_difference/mean": 1.8026823997497559, "step": 306, "step_time": 18.86735645403678 }, { "clip_ratio/high_max": 0.005217391299083829, "clip_ratio/high_mean": 0.0026086956495419145, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026086956495419145, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.5, "completions/mean_terminated_length": 17.571428298950195, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.9593222662806511, "epoch": 0.00614, "frac_reward_zero_std": 0.0, "grad_norm": 0.3924546539783478, "kl": 0.21930692763999104, "learning_rate": 8.32559643018397e-06, "loss": -0.1926, "num_tokens": 13761823.0, "reward": 0.00937499850988388, "reward_std": 0.28799980878829956, "rewards/rollout_reward_func/mean": 0.00937499850988388, "rewards/rollout_reward_func/std": 0.2976893484592438, "sampling/importance_sampling_ratio/max": 1.5784913301467896, "sampling/importance_sampling_ratio/mean": 0.5431044101715088, "sampling/importance_sampling_ratio/min": 1.401298464324817e-44, "sampling/sampling_logp_difference/max": 32.76388931274414, "sampling/sampling_logp_difference/mean": 1.526028037071228, "step": 307, "step_time": 18.645460214014747 }, { "clip_ratio/high_max": 0.005789473652839661, "clip_ratio/high_mean": 0.0028947368264198303, "clip_ratio/low_mean": 0.0034007353242486715, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006295472150668502, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.75, "completions/mean_terminated_length": 19.95652198791504, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9982441812753677, "epoch": 0.00616, "frac_reward_zero_std": 0.0, "grad_norm": 0.32225656509399414, "kl": 0.12342347903177142, "learning_rate": 8.314239606499828e-06, "loss": -0.0074, "num_tokens": 13806906.0, "reward": 0.02500000037252903, "reward_std": 0.32322707772254944, "rewards/rollout_reward_func/mean": 0.02500000037252903, "rewards/rollout_reward_func/std": 0.3172462284564972, "sampling/importance_sampling_ratio/max": 1.8345001935958862, "sampling/importance_sampling_ratio/mean": 0.4262715280056, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.07010269165039, "sampling/sampling_logp_difference/mean": 1.7931674718856812, "step": 308, "step_time": 19.632123966002837 }, { "clip_ratio/high_max": 0.011552694952115417, "clip_ratio/high_mean": 0.0064273892203345895, "clip_ratio/low_mean": 0.0015625000232830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007989889243617654, "completions/clipped_ratio": 0.3125, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 20.84375, "completions/mean_terminated_length": 19.590909957885742, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0393391102552414, "epoch": 0.00618, "frac_reward_zero_std": 0.0, "grad_norm": 0.33510681986808777, "kl": 0.45960262161679566, "learning_rate": 8.302855554099842e-06, "loss": -0.2304, "num_tokens": 13851766.0, "reward": -0.0625, "reward_std": 0.3462936282157898, "rewards/rollout_reward_func/mean": -0.0625, "rewards/rollout_reward_func/std": 0.3395918011665344, "sampling/importance_sampling_ratio/max": 1.7228553295135498, "sampling/importance_sampling_ratio/mean": 0.4285816550254822, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.820106506347656, "sampling/sampling_logp_difference/mean": 1.9127397537231445, "step": 309, "step_time": 19.563888035074342 }, { "clip_ratio/high_max": 0.007513973629102111, "clip_ratio/high_mean": 0.0037569868145510554, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005401723668910563, "completions/clipped_ratio": 0.3125, "completions/max_length": 48.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 23.0625, "completions/mean_terminated_length": 20.68181800842285, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.10636568069458, "epoch": 0.0062, "frac_reward_zero_std": 0.0, "grad_norm": 0.4099583923816681, "kl": 0.10493512358516455, "learning_rate": 8.29144442314736e-06, "loss": -0.0713, "num_tokens": 13898215.0, "reward": -0.046875, "reward_std": 0.2556631565093994, "rewards/rollout_reward_func/mean": -0.046875, "rewards/rollout_reward_func/std": 0.27472493052482605, "sampling/importance_sampling_ratio/max": 2.0215680599212646, "sampling/importance_sampling_ratio/mean": 0.4257442355155945, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.648231506347656, "sampling/sampling_logp_difference/mean": 1.8788673877716064, "step": 310, "step_time": 20.452950456994586 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.375, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 20.1875, "completions/mean_terminated_length": 17.55000114440918, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0215262025594711, "epoch": 0.00622, "frac_reward_zero_std": 0.0, "grad_norm": 0.3073941767215729, "kl": 0.20372334553394467, "learning_rate": 8.280006364162915e-06, "loss": -0.1143, "num_tokens": 13943820.0, "reward": -0.05937500298023224, "reward_std": 0.29858487844467163, "rewards/rollout_reward_func/mean": -0.05937500298023224, "rewards/rollout_reward_func/std": 0.288331538438797, "sampling/importance_sampling_ratio/max": 1.6457910537719727, "sampling/importance_sampling_ratio/mean": 0.5489632487297058, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.8987922668457, "sampling/sampling_logp_difference/mean": 1.6913217306137085, "step": 311, "step_time": 19.468712526984746 }, { "clip_ratio/high_max": 0.0059523810632526875, "clip_ratio/high_mean": 0.0029761905316263437, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029761905316263437, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.5, "completions/mean_terminated_length": 18.238096237182617, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1264383532106876, "epoch": 0.00624, "frac_reward_zero_std": 0.0, "grad_norm": 0.4628443419933319, "kl": 0.21485444298014045, "learning_rate": 8.268541528022238e-06, "loss": -0.1232, "num_tokens": 13989181.0, "reward": -0.04375000298023224, "reward_std": 0.25362101197242737, "rewards/rollout_reward_func/mean": -0.04375000298023224, "rewards/rollout_reward_func/std": 0.2804805636405945, "sampling/importance_sampling_ratio/max": 2.2540829181671143, "sampling/importance_sampling_ratio/mean": 0.5054395794868469, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.688987731933594, "sampling/sampling_logp_difference/mean": 1.998037338256836, "step": 312, "step_time": 19.037386406052974 }, { "clip_ratio/high_max": 0.0065972222946584225, "clip_ratio/high_mean": 0.0032986111473292112, "clip_ratio/low_mean": 0.0025606187991797924, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005859229946509004, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 18.8125, "completions/mean_terminated_length": 17.416667938232422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0304285530000925, "epoch": 0.00626, "frac_reward_zero_std": 0.0, "grad_norm": 0.3842003047466278, "kl": 0.22064805449917912, "learning_rate": 8.257050065954267e-06, "loss": -0.1062, "num_tokens": 14034865.0, "reward": -0.02499999664723873, "reward_std": 0.2394094467163086, "rewards/rollout_reward_func/mean": -0.02499999664723873, "rewards/rollout_reward_func/std": 0.25272706151008606, "sampling/importance_sampling_ratio/max": 1.6263607740402222, "sampling/importance_sampling_ratio/mean": 0.4724472165107727, "sampling/importance_sampling_ratio/min": 3.699427945817517e-43, "sampling/sampling_logp_difference/max": 32.778831481933594, "sampling/sampling_logp_difference/mean": 1.9573748111724854, "step": 313, "step_time": 19.25855406302435 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0014880952658131719, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026455026818439364, "completions/clipped_ratio": 0.28125, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 19.5, "completions/mean_terminated_length": 18.34782600402832, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9281301740556955, "epoch": 0.00628, "frac_reward_zero_std": 0.0, "grad_norm": 0.3295676112174988, "kl": 0.1992804470937699, "learning_rate": 8.245532129539153e-06, "loss": -0.0825, "num_tokens": 14079044.0, "reward": 0.0062499940395355225, "reward_std": 0.3800346851348877, "rewards/rollout_reward_func/mean": 0.0062499940395355225, "rewards/rollout_reward_func/std": 0.38347938656806946, "sampling/importance_sampling_ratio/max": 1.8011305332183838, "sampling/importance_sampling_ratio/mean": 0.6083611249923706, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.91424560546875, "sampling/sampling_logp_difference/mean": 1.5905380249023438, "step": 314, "step_time": 19.844761996952002 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.002652801340445876, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004011497017927468, "completions/clipped_ratio": 0.25, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.03125, "completions/mean_terminated_length": 15.916666984558105, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9634651355445385, "epoch": 0.0063, "frac_reward_zero_std": 0.0, "grad_norm": 0.7553901672363281, "kl": 0.26491956366226077, "learning_rate": 8.233987870706267e-06, "loss": 0.0461, "num_tokens": 14124810.0, "reward": -0.03749999776482582, "reward_std": 0.3382180333137512, "rewards/rollout_reward_func/mean": -0.03749999776482582, "rewards/rollout_reward_func/std": 0.35988351702690125, "sampling/importance_sampling_ratio/max": 2.6985769271850586, "sampling/importance_sampling_ratio/mean": 0.47411203384399414, "sampling/importance_sampling_ratio/min": 1.401298464324817e-44, "sampling/sampling_logp_difference/max": 32.72633361816406, "sampling/sampling_logp_difference/mean": 1.7774596214294434, "step": 315, "step_time": 18.817031971993856 }, { "clip_ratio/high_max": 0.004924242617562413, "clip_ratio/high_mean": 0.0024621213087812066, "clip_ratio/low_mean": 0.002016128972172737, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004478250280953944, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 20.8125, "completions/mean_terminated_length": 18.959999084472656, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.1122347973287106, "epoch": 0.00632, "frac_reward_zero_std": 0.0, "grad_norm": 0.4072074294090271, "kl": 0.5750970467925072, "learning_rate": 8.222417441732182e-06, "loss": -0.2519, "num_tokens": 14169751.0, "reward": -0.0624999962747097, "reward_std": 0.3180319666862488, "rewards/rollout_reward_func/mean": -0.0624999962747097, "rewards/rollout_reward_func/std": 0.3159726560115814, "sampling/importance_sampling_ratio/max": 1.9177852869033813, "sampling/importance_sampling_ratio/mean": 0.4435375928878784, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.528839111328125, "sampling/sampling_logp_difference/mean": 2.1286792755126953, "step": 316, "step_time": 20.11253789102193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.28125, "completions/mean_terminated_length": 17.518518447875977, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9783687833696604, "epoch": 0.00634, "frac_reward_zero_std": 0.0, "grad_norm": 0.2883872091770172, "kl": 0.2084206216968596, "learning_rate": 8.210820995238682e-06, "loss": -0.0955, "num_tokens": 14215086.0, "reward": 0.07500000298023224, "reward_std": 0.23402175307273865, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.26639771461486816, "sampling/importance_sampling_ratio/max": 2.0304384231567383, "sampling/importance_sampling_ratio/mean": 0.512077808380127, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.532745361328125, "sampling/sampling_logp_difference/mean": 1.9009735584259033, "step": 317, "step_time": 18.673588886027574 }, { "clip_ratio/high_max": 0.004807692486792803, "clip_ratio/high_mean": 0.0024038462433964014, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003653846215456724, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.9375, "completions/mean_terminated_length": 18.666667938232422, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9484811052680016, "epoch": 0.00636, "frac_reward_zero_std": 0.0, "grad_norm": 0.21421457827091217, "kl": 0.14959379215724766, "learning_rate": 8.199198684190737e-06, "loss": -0.185, "num_tokens": 14259470.0, "reward": 0.09062500298023224, "reward_std": 0.28861191868782043, "rewards/rollout_reward_func/mean": 0.09062500298023224, "rewards/rollout_reward_func/std": 0.28211113810539246, "sampling/importance_sampling_ratio/max": 1.8092886209487915, "sampling/importance_sampling_ratio/mean": 0.41078633069992065, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.586952209472656, "sampling/sampling_logp_difference/mean": 1.833362340927124, "step": 318, "step_time": 19.489375239034416 }, { "clip_ratio/high_max": 0.001953125, "clip_ratio/high_mean": 0.0009765625, "clip_ratio/low_mean": 0.006871693301945925, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007848255801945925, "completions/clipped_ratio": 0.28125, "completions/max_length": 44.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 22.1875, "completions/mean_terminated_length": 20.869565963745117, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.2824860699474812, "epoch": 0.00638, "frac_reward_zero_std": 0.0, "grad_norm": 0.26192623376846313, "kl": 0.1579336472786963, "learning_rate": 8.187550661894482e-06, "loss": -0.0413, "num_tokens": 14304568.0, "reward": -0.02499999850988388, "reward_std": 0.35587620735168457, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.3436051607131958, "sampling/importance_sampling_ratio/max": 1.9933876991271973, "sampling/importance_sampling_ratio/mean": 0.3469342291355133, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.64943313598633, "sampling/sampling_logp_difference/mean": 2.399493455886841, "step": 319, "step_time": 19.846256852993974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016447368543595076, "completions/clipped_ratio": 0.3125, "completions/max_length": 38.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 21.25, "completions/mean_terminated_length": 17.772727966308594, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9806276150047779, "epoch": 0.0064, "frac_reward_zero_std": 0.0, "grad_norm": 0.43679186701774597, "kl": 0.13131496752612293, "learning_rate": 8.175877081995205e-06, "loss": 0.0429, "num_tokens": 14349855.0, "reward": -0.02187499962747097, "reward_std": 0.2318773865699768, "rewards/rollout_reward_func/mean": -0.02187499962747097, "rewards/rollout_reward_func/std": 0.2744312286376953, "sampling/importance_sampling_ratio/max": 1.7243329286575317, "sampling/importance_sampling_ratio/mean": 0.5188600420951843, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.66934585571289, "sampling/sampling_logp_difference/mean": 1.6896601915359497, "step": 320, "step_time": 18.647976585969445 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.004787181271240115, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006037181243300438, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.96875, "completions/mean_terminated_length": 17.04347801208496, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.93146051466465, "epoch": 0.00642, "frac_reward_zero_std": 0.0, "grad_norm": 0.3637600243091583, "kl": 0.07972185174003243, "learning_rate": 8.16417809847532e-06, "loss": -0.1757, "num_tokens": 14393672.0, "reward": 0.02499999850988388, "reward_std": 0.3249208629131317, "rewards/rollout_reward_func/mean": 0.02499999850988388, "rewards/rollout_reward_func/std": 0.32028213143348694, "sampling/importance_sampling_ratio/max": 2.1330933570861816, "sampling/importance_sampling_ratio/mean": 0.6638242602348328, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.09427261352539, "sampling/sampling_logp_difference/mean": 1.7562682628631592, "step": 321, "step_time": 19.676621096004965 }, { "clip_ratio/high_max": 0.0026041667442768812, "clip_ratio/high_mean": 0.0013020833721384406, "clip_ratio/low_mean": 0.0030307112028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004332794575020671, "completions/clipped_ratio": 0.15625, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 18.5, "completions/mean_terminated_length": 17.77777862548828, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7979807686060667, "epoch": 0.00644, "frac_reward_zero_std": 0.0, "grad_norm": 0.38637563586235046, "kl": 0.30989341461099684, "learning_rate": 8.152453865652327e-06, "loss": -0.0632, "num_tokens": 14437309.0, "reward": 0.06562500447034836, "reward_std": 0.2710142135620117, "rewards/rollout_reward_func/mean": 0.06562500447034836, "rewards/rollout_reward_func/std": 0.2823968529701233, "sampling/importance_sampling_ratio/max": 1.5437169075012207, "sampling/importance_sampling_ratio/mean": 0.6277526617050171, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.58694076538086, "sampling/sampling_logp_difference/mean": 1.4803252220153809, "step": 322, "step_time": 18.81511138001224 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012019231216982007, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.90625, "completions/mean_terminated_length": 17.34782600402832, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.994187455624342, "epoch": 0.00646, "frac_reward_zero_std": 0.0, "grad_norm": 0.4680171310901642, "kl": 0.4498450974933803, "learning_rate": 8.140704538176782e-06, "loss": -0.0641, "num_tokens": 14481696.0, "reward": -0.03125, "reward_std": 0.3199036717414856, "rewards/rollout_reward_func/mean": -0.03125, "rewards/rollout_reward_func/std": 0.33160167932510376, "sampling/importance_sampling_ratio/max": 2.3455212116241455, "sampling/importance_sampling_ratio/mean": 0.5201457738876343, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.36819076538086, "sampling/sampling_logp_difference/mean": 1.7474819421768188, "step": 323, "step_time": 19.19350992901309 }, { "clip_ratio/high_max": 0.0031250000465661287, "clip_ratio/high_mean": 0.0015625000232830644, "clip_ratio/low_mean": 0.0079079435672611, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009470443590544164, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 20.03125, "completions/mean_terminated_length": 17.479999542236328, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.0990374609827995, "epoch": 0.00648, "frac_reward_zero_std": 0.0, "grad_norm": 0.2944188714027405, "kl": 0.398097951663658, "learning_rate": 8.128930271030256e-06, "loss": -0.1524, "num_tokens": 14526017.0, "reward": 0.02812500298023224, "reward_std": 0.29981887340545654, "rewards/rollout_reward_func/mean": 0.02812500298023224, "rewards/rollout_reward_func/std": 0.3255113661289215, "sampling/importance_sampling_ratio/max": 2.497792959213257, "sampling/importance_sampling_ratio/mean": 0.4637765884399414, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.3889045715332, "sampling/sampling_logp_difference/mean": 2.2264552116394043, "step": 324, "step_time": 19.17874547698011 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0038726478815078735, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005122647853568196, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.75, "completions/mean_terminated_length": 16.90909194946289, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9295491762459278, "epoch": 0.0065, "frac_reward_zero_std": 0.0, "grad_norm": 0.3139299154281616, "kl": 0.3029693360440433, "learning_rate": 8.117131219523293e-06, "loss": -0.0888, "num_tokens": 14570814.0, "reward": -0.05000000074505806, "reward_std": 0.29107046127319336, "rewards/rollout_reward_func/mean": -0.05000000074505806, "rewards/rollout_reward_func/std": 0.29402655363082886, "sampling/importance_sampling_ratio/max": 1.6726094484329224, "sampling/importance_sampling_ratio/mean": 0.4528464078903198, "sampling/importance_sampling_ratio/min": 1.6114932339735396e-43, "sampling/sampling_logp_difference/max": 33.67947769165039, "sampling/sampling_logp_difference/mean": 1.5675461292266846, "step": 325, "step_time": 18.810368880993337 }, { "clip_ratio/high_max": 0.0069444444961845875, "clip_ratio/high_mean": 0.0034722222480922937, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0047222222201526165, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.625, "completions/mean_terminated_length": 17.041667938232422, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.9837510045617819, "epoch": 0.00652, "frac_reward_zero_std": 0.0, "grad_norm": 0.3672914505004883, "kl": 0.20015049260109663, "learning_rate": 8.10530753929336e-06, "loss": -0.0684, "num_tokens": 14615321.0, "reward": 1.862645149230957e-09, "reward_std": 0.3243310749530792, "rewards/rollout_reward_func/mean": 1.862645149230957e-09, "rewards/rollout_reward_func/std": 0.31826141476631165, "sampling/importance_sampling_ratio/max": 2.904683828353882, "sampling/importance_sampling_ratio/mean": 0.6089479923248291, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.382606506347656, "sampling/sampling_logp_difference/mean": 1.7397867441177368, "step": 326, "step_time": 18.807315203986946 }, { "clip_ratio/high_max": 0.01098432531580329, "clip_ratio/high_mean": 0.005492162657901645, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005492162657901645, "completions/clipped_ratio": 0.3125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 22.8125, "completions/mean_terminated_length": 20.68181800842285, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 1.2976042367517948, "epoch": 0.00654, "frac_reward_zero_std": 0.0, "grad_norm": 0.19858531653881073, "kl": 0.2126985138747841, "learning_rate": 8.093459386302788e-06, "loss": -0.0264, "num_tokens": 14661539.0, "reward": 0.03124999813735485, "reward_std": 0.32957929372787476, "rewards/rollout_reward_func/mean": 0.03124999813735485, "rewards/rollout_reward_func/std": 0.33834272623062134, "sampling/importance_sampling_ratio/max": 2.059420585632324, "sampling/importance_sampling_ratio/mean": 0.3597298860549927, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.953636169433594, "sampling/sampling_logp_difference/mean": 2.4233150482177734, "step": 327, "step_time": 19.22489824098011 }, { "clip_ratio/high_max": 0.006265664240345359, "clip_ratio/high_mean": 0.0031328321201726794, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031328321201726794, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 20.21875, "completions/mean_terminated_length": 18.520000457763672, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.107943955808878, "epoch": 0.00656, "frac_reward_zero_std": 0.0, "grad_norm": 0.4111316204071045, "kl": 0.33936823112890124, "learning_rate": 8.081586916836728e-06, "loss": -0.0153, "num_tokens": 14706744.0, "reward": 0.01249999925494194, "reward_std": 0.24676774442195892, "rewards/rollout_reward_func/mean": 0.01249999925494194, "rewards/rollout_reward_func/std": 0.28026482462882996, "sampling/importance_sampling_ratio/max": 1.760411262512207, "sampling/importance_sampling_ratio/mean": 0.44404739141464233, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.576290130615234, "sampling/sampling_logp_difference/mean": 2.0438613891601562, "step": 328, "step_time": 20.095247970995842 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003441220265813172, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.3125, "completions/mean_terminated_length": 19.434782028198242, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.0578019991517067, "epoch": 0.00658, "frac_reward_zero_std": 0.0, "grad_norm": 0.40931543707847595, "kl": 0.4473404309246689, "learning_rate": 8.069690287501078e-06, "loss": -0.1295, "num_tokens": 14750918.0, "reward": 0.03437500074505806, "reward_std": 0.2903120815753937, "rewards/rollout_reward_func/mean": 0.03437500074505806, "rewards/rollout_reward_func/std": 0.30332499742507935, "sampling/importance_sampling_ratio/max": 1.6188199520111084, "sampling/importance_sampling_ratio/mean": 0.5141280293464661, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.805686950683594, "sampling/sampling_logp_difference/mean": 1.830823540687561, "step": 329, "step_time": 18.21263680397533 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.002314814832061529, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004050925956107676, "completions/clipped_ratio": 0.125, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 18.46875, "completions/mean_terminated_length": 17.46428680419922, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.0382084716111422, "epoch": 0.0066, "frac_reward_zero_std": 0.0, "grad_norm": 0.36716195940971375, "kl": 0.41639243508689106, "learning_rate": 8.05776965522042e-06, "loss": -0.0074, "num_tokens": 14795844.0, "reward": 0.015625, "reward_std": 0.3160324990749359, "rewards/rollout_reward_func/mean": 0.015625, "rewards/rollout_reward_func/std": 0.31326520442962646, "sampling/importance_sampling_ratio/max": 2.094996213912964, "sampling/importance_sampling_ratio/mean": 0.5179949998855591, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.247596740722656, "sampling/sampling_logp_difference/mean": 2.311844825744629, "step": 330, "step_time": 19.626040340037434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 19.59375, "completions/mean_terminated_length": 18.428571701049805, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.020563654601574, "epoch": 0.00662, "frac_reward_zero_std": 0.0, "grad_norm": 0.27273842692375183, "kl": 0.097859873669222, "learning_rate": 8.045825177235952e-06, "loss": -0.1531, "num_tokens": 14840685.0, "reward": -0.0062499940395355225, "reward_std": 0.32129037380218506, "rewards/rollout_reward_func/mean": -0.0062499940395355225, "rewards/rollout_reward_func/std": 0.3435465097427368, "sampling/importance_sampling_ratio/max": 1.6186566352844238, "sampling/importance_sampling_ratio/mean": 0.496154248714447, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.78618621826172, "sampling/sampling_logp_difference/mean": 1.7674243450164795, "step": 331, "step_time": 19.573769835042185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0029825499514117837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029825499514117837, "completions/clipped_ratio": 0.21875, "completions/max_length": 54.0, "completions/max_terminated_length": 54.0, "completions/mean_length": 17.6875, "completions/mean_terminated_length": 17.03999900817871, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.8507928587496281, "epoch": 0.00664, "frac_reward_zero_std": 0.0, "grad_norm": 0.5036233067512512, "kl": 0.0806203291285783, "learning_rate": 8.033857011103411e-06, "loss": -0.118, "num_tokens": 14884075.0, "reward": 0.05000000074505806, "reward_std": 0.2610272169113159, "rewards/rollout_reward_func/mean": 0.05000000074505806, "rewards/rollout_reward_func/std": 0.2602728009223938, "sampling/importance_sampling_ratio/max": 1.8309956789016724, "sampling/importance_sampling_ratio/mean": 0.7012410163879395, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.493186950683594, "sampling/sampling_logp_difference/mean": 1.6313655376434326, "step": 332, "step_time": 18.635870208017877 }, { "clip_ratio/high_max": 0.005321558099240065, "clip_ratio/high_mean": 0.003579896758310497, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003579896758310497, "completions/clipped_ratio": 0.21875, "completions/max_length": 34.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 18.8125, "completions/mean_terminated_length": 19.15999984741211, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0347192268818617, "epoch": 0.00666, "frac_reward_zero_std": 0.0, "grad_norm": 0.3608863353729248, "kl": 0.197649285197258, "learning_rate": 8.021865314691006e-06, "loss": -0.1468, "num_tokens": 14929014.0, "reward": -0.006250001490116119, "reward_std": 0.2527860403060913, "rewards/rollout_reward_func/mean": -0.006250001490116119, "rewards/rollout_reward_func/std": 0.2758418321609497, "sampling/importance_sampling_ratio/max": 1.4513471126556396, "sampling/importance_sampling_ratio/mean": 0.5267225503921509, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.326393127441406, "sampling/sampling_logp_difference/mean": 2.1461129188537598, "step": 333, "step_time": 19.789535664996947 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.00831923563964665, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009476643055677414, "completions/clipped_ratio": 0.25, "completions/max_length": 42.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 18.84375, "completions/mean_terminated_length": 17.45833396911621, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.142771240323782, "epoch": 0.00668, "frac_reward_zero_std": 0.0, "grad_norm": 0.46993735432624817, "kl": 0.13321864022873342, "learning_rate": 8.009850246177311e-06, "loss": -0.145, "num_tokens": 14974062.0, "reward": 0.01875000074505806, "reward_std": 0.30560925602912903, "rewards/rollout_reward_func/mean": 0.01875000074505806, "rewards/rollout_reward_func/std": 0.3187045454978943, "sampling/importance_sampling_ratio/max": 1.8366998434066772, "sampling/importance_sampling_ratio/mean": 0.547290563583374, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.14448165893555, "sampling/sampling_logp_difference/mean": 2.0911898612976074, "step": 334, "step_time": 18.832056837025448 }, { "clip_ratio/high_max": 0.005972222192212939, "clip_ratio/high_mean": 0.0029861110961064696, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004143518512137234, "completions/clipped_ratio": 0.1875, "completions/max_length": 48.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.09375, "completions/mean_terminated_length": 14.884615898132324, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8909776490181684, "epoch": 0.0067, "frac_reward_zero_std": 0.0, "grad_norm": 0.3156927227973938, "kl": 0.8127670949324965, "learning_rate": 7.997811964049207e-06, "loss": 0.0204, "num_tokens": 15016966.0, "reward": 0.037500008940696716, "reward_std": 0.31347721815109253, "rewards/rollout_reward_func/mean": 0.037500008940696716, "rewards/rollout_reward_func/std": 0.34242963790893555, "sampling/importance_sampling_ratio/max": 1.3622092008590698, "sampling/importance_sampling_ratio/mean": 0.5683027505874634, "sampling/importance_sampling_ratio/min": 8.547920632381384e-44, "sampling/sampling_logp_difference/max": 32.985862731933594, "sampling/sampling_logp_difference/mean": 1.6632750034332275, "step": 335, "step_time": 18.705069692019606 }, { "clip_ratio/high_max": 0.008826958481222391, "clip_ratio/high_mean": 0.0044134792406111956, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005772174918092787, "completions/clipped_ratio": 0.25, "completions/max_length": 36.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 19.84375, "completions/mean_terminated_length": 16.45833396911621, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.111448934301734, "epoch": 0.00672, "frac_reward_zero_std": 0.0, "grad_norm": 0.3582610487937927, "kl": 0.12786736921407282, "learning_rate": 7.98575062709977e-06, "loss": -0.0953, "num_tokens": 15062877.0, "reward": -0.11562500149011612, "reward_std": 0.3705013394355774, "rewards/rollout_reward_func/mean": -0.11562500149011612, "rewards/rollout_reward_func/std": 0.37512093782424927, "sampling/importance_sampling_ratio/max": 2.6730377674102783, "sampling/importance_sampling_ratio/mean": 0.4601829946041107, "sampling/importance_sampling_ratio/min": 5.605193857299268e-45, "sampling/sampling_logp_difference/max": 32.851348876953125, "sampling/sampling_logp_difference/mean": 2.035456418991089, "step": 336, "step_time": 19.885388854978373 }, { "clip_ratio/high_max": 0.006770833628252149, "clip_ratio/high_mean": 0.0033854168141260743, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033854168141260743, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.6875, "completions/mean_terminated_length": 19.0, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.126038085669279, "epoch": 0.00674, "frac_reward_zero_std": 0.0, "grad_norm": 0.3237841725349426, "kl": 0.1708326981170103, "learning_rate": 7.97366639442619e-06, "loss": -0.1927, "num_tokens": 15108026.0, "reward": -0.046875, "reward_std": 0.31414294242858887, "rewards/rollout_reward_func/mean": -0.046875, "rewards/rollout_reward_func/std": 0.3582973778247833, "sampling/importance_sampling_ratio/max": 1.6569926738739014, "sampling/importance_sampling_ratio/mean": 0.4577881693840027, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 33.02444076538086, "sampling/sampling_logp_difference/mean": 1.8753881454467773, "step": 337, "step_time": 19.02827923397126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.21875, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 18.375, "completions/mean_terminated_length": 18.399999618530273, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9431248009204865, "epoch": 0.00676, "frac_reward_zero_std": 0.0, "grad_norm": 0.3792690336704254, "kl": 0.12010994087904692, "learning_rate": 7.961559425427666e-06, "loss": -0.2046, "num_tokens": 15152198.0, "reward": -0.02499999850988388, "reward_std": 0.3155513107776642, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.32824066281318665, "sampling/importance_sampling_ratio/max": 1.7515738010406494, "sampling/importance_sampling_ratio/mean": 0.5681446194648743, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 33.12259292602539, "sampling/sampling_logp_difference/mean": 1.5963842868804932, "step": 338, "step_time": 19.023081776991603 }, { "clip_ratio/high_max": 0.018900427501648664, "clip_ratio/high_mean": 0.009450213750824332, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009450213750824332, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 18.4375, "completions/mean_terminated_length": 17.239999771118164, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0609029568731785, "epoch": 0.00678, "frac_reward_zero_std": 0.0, "grad_norm": 0.4861160218715668, "kl": 0.11192182009108365, "learning_rate": 7.949429879803298e-06, "loss": -0.0738, "num_tokens": 15197210.0, "reward": 0.009375001303851604, "reward_std": 0.31481653451919556, "rewards/rollout_reward_func/mean": 0.009375001303851604, "rewards/rollout_reward_func/std": 0.3155736029148102, "sampling/importance_sampling_ratio/max": 2.5144028663635254, "sampling/importance_sampling_ratio/mean": 0.5348716974258423, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.922386169433594, "sampling/sampling_logp_difference/mean": 2.08553147315979, "step": 339, "step_time": 18.673733377014287 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.007003206294029951, "clip_ratio/low_min": 0.0016891892300918698, "clip_ratio/region_mean": 0.008160613710060716, "completions/clipped_ratio": 0.03125, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 17.03125, "completions/mean_terminated_length": 16.70967674255371, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.931223364546895, "epoch": 0.0068, "frac_reward_zero_std": 0.0, "grad_norm": 0.32434597611427307, "kl": 0.09879016876220703, "learning_rate": 7.937277917549998e-06, "loss": -0.2093, "num_tokens": 15240983.0, "reward": 0.053125008940696716, "reward_std": 0.2866469919681549, "rewards/rollout_reward_func/mean": 0.053125008940696716, "rewards/rollout_reward_func/std": 0.28958743810653687, "sampling/importance_sampling_ratio/max": 1.472671627998352, "sampling/importance_sampling_ratio/mean": 0.5918258428573608, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.794898986816406, "sampling/sampling_logp_difference/mean": 1.9835398197174072, "step": 340, "step_time": 19.895805233027204 }, { "clip_ratio/high_max": 0.010667067486792803, "clip_ratio/high_mean": 0.005333533743396401, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006978270597755909, "completions/clipped_ratio": 0.34375, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 20.75, "completions/mean_terminated_length": 18.14285659790039, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0111047122627497, "epoch": 0.00682, "frac_reward_zero_std": 0.0, "grad_norm": 0.26989349722862244, "kl": 0.15432367753237486, "learning_rate": 7.925103698960361e-06, "loss": -0.1477, "num_tokens": 15286369.0, "reward": -0.006249994970858097, "reward_std": 0.32207372784614563, "rewards/rollout_reward_func/mean": -0.006249994970858097, "rewards/rollout_reward_func/std": 0.35005760192871094, "sampling/importance_sampling_ratio/max": 1.3267337083816528, "sampling/importance_sampling_ratio/mean": 0.4823097586631775, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.38890075683594, "sampling/sampling_logp_difference/mean": 1.5244121551513672, "step": 341, "step_time": 19.356987359002233 }, { "clip_ratio/high_max": 0.003289473708719015, "clip_ratio/high_mean": 0.0016447368543595076, "clip_ratio/low_mean": 0.0029956427169963717, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004640379571355879, "completions/clipped_ratio": 0.09375, "completions/max_length": 43.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 19.21875, "completions/mean_terminated_length": 19.172412872314453, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1377887949347496, "epoch": 0.00684, "frac_reward_zero_std": 0.0, "grad_norm": 0.2823033630847931, "kl": 0.32835609233006835, "learning_rate": 7.91290738462056e-06, "loss": -0.1033, "num_tokens": 15331260.0, "reward": -0.04062499850988388, "reward_std": 0.35827571153640747, "rewards/rollout_reward_func/mean": -0.04062499850988388, "rewards/rollout_reward_func/std": 0.36354827880859375, "sampling/importance_sampling_ratio/max": 1.04604971408844, "sampling/importance_sampling_ratio/mean": 0.3034703731536865, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.497589111328125, "sampling/sampling_logp_difference/mean": 2.33135986328125, "step": 342, "step_time": 19.043143571034307 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.0012019231216982007, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026900183875113726, "completions/clipped_ratio": 0.1875, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 19.6875, "completions/mean_terminated_length": 19.307693481445312, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0773554407060146, "epoch": 0.00686, "frac_reward_zero_std": 0.0, "grad_norm": 0.2662215530872345, "kl": 0.33032115595415235, "learning_rate": 7.900689135408226e-06, "loss": -0.1187, "num_tokens": 15376144.0, "reward": -0.0031249970197677612, "reward_std": 0.2808924913406372, "rewards/rollout_reward_func/mean": -0.0031249970197677612, "rewards/rollout_reward_func/std": 0.3084658980369568, "sampling/importance_sampling_ratio/max": 1.9558873176574707, "sampling/importance_sampling_ratio/mean": 0.43547356128692627, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.09427261352539, "sampling/sampling_logp_difference/mean": 2.1160411834716797, "step": 343, "step_time": 19.473465564005892 }, { "clip_ratio/high_max": 0.006914511555805802, "clip_ratio/high_mean": 0.003457255777902901, "clip_ratio/low_mean": 0.0035309437080286443, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006988199427723885, "completions/clipped_ratio": 0.25, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 19.8125, "completions/mean_terminated_length": 18.625, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.086434157565236, "epoch": 0.00688, "frac_reward_zero_std": 0.0, "grad_norm": 0.32911503314971924, "kl": 0.14197400643024594, "learning_rate": 7.888449112490325e-06, "loss": -0.181, "num_tokens": 15420020.0, "reward": 0.10000000894069672, "reward_std": 0.3058144152164459, "rewards/rollout_reward_func/mean": 0.10000000894069672, "rewards/rollout_reward_func/std": 0.3026762306690216, "sampling/importance_sampling_ratio/max": 2.155996799468994, "sampling/importance_sampling_ratio/mean": 0.5198014378547668, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.04170608520508, "sampling/sampling_logp_difference/mean": 2.11352276802063, "step": 344, "step_time": 19.199490158978733 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012499999720603228, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 19.6875, "completions/mean_terminated_length": 18.239999771118164, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9854354541748762, "epoch": 0.0069, "frac_reward_zero_std": 0.0, "grad_norm": 0.26480743288993835, "kl": 0.2380663980729878, "learning_rate": 7.876187477321033e-06, "loss": -0.1877, "num_tokens": 15465146.0, "reward": -0.031249992549419403, "reward_std": 0.31151747703552246, "rewards/rollout_reward_func/mean": -0.031249992549419403, "rewards/rollout_reward_func/std": 0.36049923300743103, "sampling/importance_sampling_ratio/max": 1.7853137254714966, "sampling/importance_sampling_ratio/mean": 0.4770602285861969, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.70698547363281, "sampling/sampling_logp_difference/mean": 1.921879529953003, "step": 345, "step_time": 19.79552716197213 }, { "clip_ratio/high_max": 0.004807692486792803, "clip_ratio/high_mean": 0.0024038462433964014, "clip_ratio/low_mean": 0.010327404597774148, "clip_ratio/low_min": 0.0024038462433964014, "clip_ratio/region_mean": 0.01273125084117055, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 15.3125, "completions/mean_terminated_length": 14.642857551574707, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9349827971309423, "epoch": 0.00692, "frac_reward_zero_std": 0.0, "grad_norm": 0.3958936333656311, "kl": 0.273370654322207, "learning_rate": 7.86390439163961e-06, "loss": -0.1099, "num_tokens": 15508701.0, "reward": 0.012500002980232239, "reward_std": 0.2895490229129791, "rewards/rollout_reward_func/mean": 0.012500002980232239, "rewards/rollout_reward_func/std": 0.3159726560115814, "sampling/importance_sampling_ratio/max": 1.4441401958465576, "sampling/importance_sampling_ratio/mean": 0.5610843300819397, "sampling/importance_sampling_ratio/min": 2.396220373995437e-43, "sampling/sampling_logp_difference/max": 33.438018798828125, "sampling/sampling_logp_difference/mean": 1.6833431720733643, "step": 346, "step_time": 18.11148191402026 }, { "clip_ratio/high_max": 0.004949534311890602, "clip_ratio/high_mean": 0.002474767155945301, "clip_ratio/low_mean": 0.0026704545598477125, "clip_ratio/low_min": 0.0024999999441206455, "clip_ratio/region_mean": 0.005145221715793014, "completions/clipped_ratio": 0.28125, "completions/max_length": 28.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.375, "completions/mean_terminated_length": 17.39130401611328, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0442831441760063, "epoch": 0.00694, "frac_reward_zero_std": 0.0, "grad_norm": 0.2898387014865875, "kl": 0.11508413427509367, "learning_rate": 7.851600017468257e-06, "loss": -0.1291, "num_tokens": 15553121.0, "reward": 0.07187500596046448, "reward_std": 0.31045031547546387, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.3061038851737976, "sampling/importance_sampling_ratio/max": 1.581270456314087, "sampling/importance_sampling_ratio/mean": 0.5737898945808411, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.138206481933594, "sampling/sampling_logp_difference/mean": 2.1628174781799316, "step": 347, "step_time": 18.49619480399997 }, { "clip_ratio/high_max": 0.008333333767950535, "clip_ratio/high_mean": 0.004166666883975267, "clip_ratio/low_mean": 0.0027225379599258304, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006889204843901098, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 19.25, "completions/mean_terminated_length": 16.173913955688477, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.082832921296358, "epoch": 0.00696, "frac_reward_zero_std": 0.0, "grad_norm": 0.35366180539131165, "kl": 0.21591190958861262, "learning_rate": 7.839274517109987e-06, "loss": -0.0907, "num_tokens": 15598310.0, "reward": -0.03437499701976776, "reward_std": 0.37398630380630493, "rewards/rollout_reward_func/mean": -0.03437499701976776, "rewards/rollout_reward_func/std": 0.36421310901641846, "sampling/importance_sampling_ratio/max": 1.566945195198059, "sampling/importance_sampling_ratio/mean": 0.36329185962677, "sampling/importance_sampling_ratio/min": 2.474693087997627e-42, "sampling/sampling_logp_difference/max": 33.09427261352539, "sampling/sampling_logp_difference/mean": 2.1045994758605957, "step": 348, "step_time": 19.553158705995884 }, { "clip_ratio/high_max": 0.001953125, "clip_ratio/high_mean": 0.0009765625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009765625, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.125, "completions/mean_terminated_length": 16.740741729736328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.932849669829011, "epoch": 0.00698, "frac_reward_zero_std": 0.0, "grad_norm": 0.3325035572052002, "kl": 0.2821707301773131, "learning_rate": 7.826928053146482e-06, "loss": -0.1392, "num_tokens": 15642980.0, "reward": 0.01562500186264515, "reward_std": 0.26365384459495544, "rewards/rollout_reward_func/mean": 0.01562500186264515, "rewards/rollout_reward_func/std": 0.27369558811187744, "sampling/importance_sampling_ratio/max": 1.290751338005066, "sampling/importance_sampling_ratio/mean": 0.5195313692092896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.77930450439453, "sampling/sampling_logp_difference/mean": 1.7255315780639648, "step": 349, "step_time": 17.92696479598817 }, { "clip_ratio/high_max": 0.007305195089429617, "clip_ratio/high_mean": 0.0036525975447148085, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036525975447148085, "completions/clipped_ratio": 0.28125, "completions/max_length": 56.0, "completions/max_terminated_length": 56.0, "completions/mean_length": 20.875, "completions/mean_terminated_length": 18.782609939575195, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3005354851484299, "epoch": 0.007, "frac_reward_zero_std": 0.0, "grad_norm": 0.46845415234565735, "kl": 0.1654057571431622, "learning_rate": 7.814560788435947e-06, "loss": -0.2268, "num_tokens": 15687942.0, "reward": -0.06562499701976776, "reward_std": 0.37120556831359863, "rewards/rollout_reward_func/mean": -0.06562499701976776, "rewards/rollout_reward_func/std": 0.3712310194969177, "sampling/importance_sampling_ratio/max": 2.3120763301849365, "sampling/importance_sampling_ratio/mean": 0.4555455148220062, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.63889694213867, "sampling/sampling_logp_difference/mean": 2.658445358276367, "step": 350, "step_time": 20.74381178097974 }, { "clip_ratio/high_max": 0.0064062499441206455, "clip_ratio/high_mean": 0.0032031249720603228, "clip_ratio/low_mean": 0.00341191072948277, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006615035701543093, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.78125, "completions/mean_terminated_length": 18.39130401611328, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1815837025642395, "epoch": 0.00702, "frac_reward_zero_std": 0.0, "grad_norm": 0.2881457507610321, "kl": 0.42485486157238483, "learning_rate": 7.802172886110968e-06, "loss": -0.1121, "num_tokens": 15733581.0, "reward": -0.05937499925494194, "reward_std": 0.2973645031452179, "rewards/rollout_reward_func/mean": -0.05937499925494194, "rewards/rollout_reward_func/std": 0.311975359916687, "sampling/importance_sampling_ratio/max": 1.365574836730957, "sampling/importance_sampling_ratio/mean": 0.4734727740287781, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.8513298034668, "sampling/sampling_logp_difference/mean": 2.1482558250427246, "step": 351, "step_time": 18.885183161022724 }, { "clip_ratio/high_max": 0.006964285857975483, "clip_ratio/high_mean": 0.0034821429289877415, "clip_ratio/low_mean": 0.0008012820617295802, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004283424990717322, "completions/clipped_ratio": 0.15625, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 17.78125, "completions/mean_terminated_length": 17.518518447875977, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0491465013474226, "epoch": 0.00704, "frac_reward_zero_std": 0.0, "grad_norm": 0.32919570803642273, "kl": 0.33300311397761106, "learning_rate": 7.789764509576347e-06, "loss": 0.0221, "num_tokens": 15778142.0, "reward": -0.056249991059303284, "reward_std": 0.2832104563713074, "rewards/rollout_reward_func/mean": -0.056249991059303284, "rewards/rollout_reward_func/std": 0.3161640167236328, "sampling/importance_sampling_ratio/max": 1.3305872678756714, "sampling/importance_sampling_ratio/mean": 0.4983592629432678, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.48359298706055, "sampling/sampling_logp_difference/mean": 1.9994429349899292, "step": 352, "step_time": 19.346571247981046 }, { "clip_ratio/high_max": 0.0022321429569274187, "clip_ratio/high_mean": 0.0011160714784637094, "clip_ratio/low_mean": 0.0032552083721384406, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00437127985060215, "completions/clipped_ratio": 0.125, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 16.375, "completions/mean_terminated_length": 15.571429252624512, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0509655559435487, "epoch": 0.00706, "frac_reward_zero_std": 0.0, "grad_norm": 0.30178555846214294, "kl": 0.20922063919715583, "learning_rate": 7.77733582250696e-06, "loss": -0.1555, "num_tokens": 15821818.0, "reward": 0.02500000037252903, "reward_std": 0.29607129096984863, "rewards/rollout_reward_func/mean": 0.02500000037252903, "rewards/rollout_reward_func/std": 0.29729968309402466, "sampling/importance_sampling_ratio/max": 1.3642374277114868, "sampling/importance_sampling_ratio/mean": 0.5530126094818115, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.09424591064453, "sampling/sampling_logp_difference/mean": 2.2597224712371826, "step": 353, "step_time": 18.048896396969212 }, { "clip_ratio/high_max": 0.0026041667442768812, "clip_ratio/high_mean": 0.0013020833721384406, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025520833441987634, "completions/clipped_ratio": 0.03125, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 16.78125, "completions/mean_terminated_length": 16.290321350097656, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9871289096772671, "epoch": 0.00708, "frac_reward_zero_std": 0.0, "grad_norm": 0.26727837324142456, "kl": 0.2890445729717612, "learning_rate": 7.764886988845588e-06, "loss": -0.0792, "num_tokens": 15865603.0, "reward": 0.0625, "reward_std": 0.3138720989227295, "rewards/rollout_reward_func/mean": 0.0625, "rewards/rollout_reward_func/std": 0.3279949128627777, "sampling/importance_sampling_ratio/max": 1.7903950214385986, "sampling/importance_sampling_ratio/mean": 0.5122974514961243, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.430049896240234, "sampling/sampling_logp_difference/mean": 2.002999782562256, "step": 354, "step_time": 17.88680935796583 }, { "clip_ratio/high_max": 0.007532206131145358, "clip_ratio/high_mean": 0.003766103065572679, "clip_ratio/low_mean": 0.002536526066251099, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006302629131823778, "completions/clipped_ratio": 0.15625, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 17.5625, "completions/mean_terminated_length": 16.44444465637207, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1520336866378784, "epoch": 0.0071, "frac_reward_zero_std": 0.0, "grad_norm": 0.4987872540950775, "kl": 0.4312828052788973, "learning_rate": 7.752418172800769e-06, "loss": -0.2131, "num_tokens": 15908604.0, "reward": 0.0625, "reward_std": 0.24948273599147797, "rewards/rollout_reward_func/mean": 0.0625, "rewards/rollout_reward_func/std": 0.2697071433067322, "sampling/importance_sampling_ratio/max": 2.9178123474121094, "sampling/importance_sampling_ratio/mean": 0.6279541850090027, "sampling/importance_sampling_ratio/min": 2.6624670822171524e-44, "sampling/sampling_logp_difference/max": 33.15546798706055, "sampling/sampling_logp_difference/mean": 1.9840900897979736, "step": 355, "step_time": 19.155808345021796 }, { "clip_ratio/high_max": 0.007925724843516946, "clip_ratio/high_mean": 0.003962862421758473, "clip_ratio/low_mean": 0.0011160714784637094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005078933900222182, "completions/clipped_ratio": 0.28125, "completions/max_length": 48.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 19.8125, "completions/mean_terminated_length": 19.04347801208496, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.130358375608921, "epoch": 0.00712, "frac_reward_zero_std": 0.0, "grad_norm": 0.2452092468738556, "kl": 0.26446561655029655, "learning_rate": 7.73992953884461e-06, "loss": -0.1459, "num_tokens": 15953590.0, "reward": -0.09375, "reward_std": 0.3524979054927826, "rewards/rollout_reward_func/mean": -0.09375, "rewards/rollout_reward_func/std": 0.38515809178352356, "sampling/importance_sampling_ratio/max": 1.206689476966858, "sampling/importance_sampling_ratio/mean": 0.40110015869140625, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.24918746948242, "sampling/sampling_logp_difference/mean": 2.098198413848877, "step": 356, "step_time": 19.35801157298556 }, { "clip_ratio/high_max": 0.007532206131145358, "clip_ratio/high_mean": 0.003766103065572679, "clip_ratio/low_mean": 0.0028947368264198303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006660839891992509, "completions/clipped_ratio": 0.28125, "completions/max_length": 31.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.59375, "completions/mean_terminated_length": 18.39130401611328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3329941630363464, "epoch": 0.00714, "frac_reward_zero_std": 0.0, "grad_norm": 0.40722009539604187, "kl": 0.15892492549028248, "learning_rate": 7.727421251710637e-06, "loss": -0.0524, "num_tokens": 15998388.0, "reward": 0.015625, "reward_std": 0.2576555013656616, "rewards/rollout_reward_func/mean": 0.015625, "rewards/rollout_reward_func/std": 0.26287415623664856, "sampling/importance_sampling_ratio/max": 1.3425054550170898, "sampling/importance_sampling_ratio/mean": 0.38849085569381714, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.57066345214844, "sampling/sampling_logp_difference/mean": 2.3807578086853027, "step": 357, "step_time": 19.578770621024887 }, { "clip_ratio/high_max": 0.003289473708719015, "clip_ratio/high_mean": 0.0016447368543595076, "clip_ratio/low_mean": 0.008205671212635934, "clip_ratio/low_min": 0.0024038462433964014, "clip_ratio/region_mean": 0.009850408066995442, "completions/clipped_ratio": 0.15625, "completions/max_length": 47.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 18.0625, "completions/mean_terminated_length": 16.44444465637207, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2410971112549305, "epoch": 0.00716, "frac_reward_zero_std": 0.0, "grad_norm": 0.30354344844818115, "kl": 0.18737979419529438, "learning_rate": 7.714893476391613e-06, "loss": -0.1061, "num_tokens": 16043099.0, "reward": 0.0312500074505806, "reward_std": 0.32667529582977295, "rewards/rollout_reward_func/mean": 0.0312500074505806, "rewards/rollout_reward_func/std": 0.3257126212120056, "sampling/importance_sampling_ratio/max": 1.5573393106460571, "sampling/importance_sampling_ratio/mean": 0.4717525541782379, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.957916259765625, "sampling/sampling_logp_difference/mean": 2.7798397541046143, "step": 358, "step_time": 19.122951364028268 }, { "clip_ratio/high_max": 0.004546957788988948, "clip_ratio/high_mean": 0.002273478894494474, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0039182157488539815, "completions/clipped_ratio": 0.28125, "completions/max_length": 44.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 19.59375, "completions/mean_terminated_length": 18.478261947631836, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.2491548620164394, "epoch": 0.00718, "frac_reward_zero_std": 0.0, "grad_norm": 0.563193678855896, "kl": 0.14943003910593688, "learning_rate": 7.702346378137364e-06, "loss": -0.1356, "num_tokens": 16088713.0, "reward": -0.02812499925494194, "reward_std": 0.27857494354248047, "rewards/rollout_reward_func/mean": -0.02812499925494194, "rewards/rollout_reward_func/std": 0.27618876099586487, "sampling/importance_sampling_ratio/max": 2.8256382942199707, "sampling/importance_sampling_ratio/mean": 0.518811047077179, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.29610061645508, "sampling/sampling_logp_difference/mean": 2.1727631092071533, "step": 359, "step_time": 19.473365535988705 }, { "clip_ratio/high_max": 0.015812630066648126, "clip_ratio/high_mean": 0.007906315033324063, "clip_ratio/low_mean": 0.0028409091755747795, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010747224208898842, "completions/clipped_ratio": 0.15625, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 20.09375, "completions/mean_terminated_length": 19.22222328186035, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.1832213774323463, "epoch": 0.0072, "frac_reward_zero_std": 0.0, "grad_norm": 0.39741718769073486, "kl": 0.11832451028749347, "learning_rate": 7.689780122452598e-06, "loss": -0.1778, "num_tokens": 16134669.0, "reward": -0.028124993667006493, "reward_std": 0.3671760559082031, "rewards/rollout_reward_func/mean": -0.028124993667006493, "rewards/rollout_reward_func/std": 0.35671836137771606, "sampling/importance_sampling_ratio/max": 1.4762036800384521, "sampling/importance_sampling_ratio/mean": 0.4279298186302185, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.3841438293457, "sampling/sampling_logp_difference/mean": 2.3024656772613525, "step": 360, "step_time": 19.80061622998619 }, { "clip_ratio/high_max": 0.01822916674427688, "clip_ratio/high_mean": 0.00911458337213844, "clip_ratio/low_mean": 0.004019474727101624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013134058099240065, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.375, "completions/mean_terminated_length": 16.846155166625977, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2034492194652557, "epoch": 0.00722, "frac_reward_zero_std": 0.0, "grad_norm": 0.2597690224647522, "kl": 0.462540298467502, "learning_rate": 7.67719487509472e-06, "loss": -0.1061, "num_tokens": 16179481.0, "reward": -0.04999999701976776, "reward_std": 0.37703531980514526, "rewards/rollout_reward_func/mean": -0.04999999701976776, "rewards/rollout_reward_func/std": 0.3610021770000458, "sampling/importance_sampling_ratio/max": 1.4573192596435547, "sampling/importance_sampling_ratio/mean": 0.3553300201892853, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.904296875, "sampling/sampling_logp_difference/mean": 2.3220715522766113, "step": 361, "step_time": 18.105355832987698 }, { "clip_ratio/high_max": 0.004807692486792803, "clip_ratio/high_mean": 0.0024038462433964014, "clip_ratio/low_mean": 0.004781152296345681, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007184998539742082, "completions/clipped_ratio": 0.09375, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 16.21875, "completions/mean_terminated_length": 15.896552085876465, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1588265225291252, "epoch": 0.00724, "frac_reward_zero_std": 0.0, "grad_norm": 0.33613285422325134, "kl": 0.21239280002191663, "learning_rate": 7.664590802071653e-06, "loss": -0.213, "num_tokens": 16224064.0, "reward": -0.05000000447034836, "reward_std": 0.28312182426452637, "rewards/rollout_reward_func/mean": -0.05000000447034836, "rewards/rollout_reward_func/std": 0.3016085922718048, "sampling/importance_sampling_ratio/max": 1.4970037937164307, "sampling/importance_sampling_ratio/mean": 0.44005829095840454, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.73823928833008, "sampling/sampling_logp_difference/mean": 2.6030895709991455, "step": 362, "step_time": 19.116164063001634 }, { "clip_ratio/high_max": 0.012128851609304547, "clip_ratio/high_mean": 0.006064425804652274, "clip_ratio/low_mean": 0.0029761905316263437, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009040616336278617, "completions/clipped_ratio": 0.21875, "completions/max_length": 46.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 16.03125, "completions/mean_terminated_length": 14.079999923706055, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9923048485070467, "epoch": 0.00726, "frac_reward_zero_std": 0.0, "grad_norm": 0.36576539278030396, "kl": 0.3397710509598255, "learning_rate": 7.651968069639637e-06, "loss": -0.0602, "num_tokens": 16267851.0, "reward": -0.02187500149011612, "reward_std": 0.30289244651794434, "rewards/rollout_reward_func/mean": -0.02187500149011612, "rewards/rollout_reward_func/std": 0.3097703456878662, "sampling/importance_sampling_ratio/max": 1.9705177545547485, "sampling/importance_sampling_ratio/mean": 0.5133130550384521, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.6214714050293, "sampling/sampling_logp_difference/mean": 1.725704312324524, "step": 363, "step_time": 18.819453810006962 }, { "clip_ratio/high_max": 0.004732142901048064, "clip_ratio/high_mean": 0.002366071450524032, "clip_ratio/low_mean": 0.007552043651230633, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009918115101754665, "completions/clipped_ratio": 0.21875, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 18.90625, "completions/mean_terminated_length": 18.760000228881836, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2961262427270412, "epoch": 0.00728, "frac_reward_zero_std": 0.0, "grad_norm": 0.2523501515388489, "kl": 0.14075791276991367, "learning_rate": 7.63932684430105e-06, "loss": -0.0289, "num_tokens": 16313207.0, "reward": 0.02500000223517418, "reward_std": 0.34835225343704224, "rewards/rollout_reward_func/mean": 0.02500000223517418, "rewards/rollout_reward_func/std": 0.3436051607131958, "sampling/importance_sampling_ratio/max": 1.4252293109893799, "sampling/importance_sampling_ratio/mean": 0.30637967586517334, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.6800537109375, "sampling/sampling_logp_difference/mean": 2.690260410308838, "step": 364, "step_time": 20.14458386701881 }, { "clip_ratio/high_max": 0.005965909222140908, "clip_ratio/high_mean": 0.002982954611070454, "clip_ratio/low_mean": 0.0018382353009656072, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004821189912036061, "completions/clipped_ratio": 0.1875, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 18.5, "completions/mean_terminated_length": 18.19230842590332, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1591188684105873, "epoch": 0.0073, "frac_reward_zero_std": 0.0, "grad_norm": 0.2586943507194519, "kl": 0.13097931258380413, "learning_rate": 7.626667292802197e-06, "loss": -0.044, "num_tokens": 16358682.0, "reward": -0.02812499925494194, "reward_std": 0.2672184705734253, "rewards/rollout_reward_func/mean": -0.02812499925494194, "rewards/rollout_reward_func/std": 0.25682222843170166, "sampling/importance_sampling_ratio/max": 1.4291138648986816, "sampling/importance_sampling_ratio/mean": 0.41667935252189636, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.098541259765625, "sampling/sampling_logp_difference/mean": 2.1802518367767334, "step": 365, "step_time": 19.319127723982092 }, { "clip_ratio/high_max": 0.005444646114483476, "clip_ratio/high_mean": 0.002722323057241738, "clip_ratio/low_mean": 0.004226603894494474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006948926951736212, "completions/clipped_ratio": 0.125, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 16.3125, "completions/mean_terminated_length": 15.35714340209961, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0540347397327423, "epoch": 0.00732, "frac_reward_zero_std": 0.0, "grad_norm": 0.4039323329925537, "kl": 0.3777893171645701, "learning_rate": 7.613989582131121e-06, "loss": 0.0367, "num_tokens": 16403051.0, "reward": 0.13125000894069672, "reward_std": 0.2123754769563675, "rewards/rollout_reward_func/mean": 0.13125000894069672, "rewards/rollout_reward_func/std": 0.25328487157821655, "sampling/importance_sampling_ratio/max": 1.8904789686203003, "sampling/importance_sampling_ratio/mean": 0.5061306953430176, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.01215362548828, "sampling/sampling_logp_difference/mean": 2.437593936920166, "step": 366, "step_time": 18.238285017985618 }, { "clip_ratio/high_max": 0.01064814836718142, "clip_ratio/high_mean": 0.00532407418359071, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006682769861072302, "completions/clipped_ratio": 0.03125, "completions/max_length": 30.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 14.75, "completions/mean_terminated_length": 14.709676742553711, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9822716787457466, "epoch": 0.00734, "frac_reward_zero_std": 0.0, "grad_norm": 0.30424898862838745, "kl": 0.21291832299903035, "learning_rate": 7.6012938795153966e-06, "loss": -0.0242, "num_tokens": 16446881.0, "reward": 0.12187500298023224, "reward_std": 0.29665231704711914, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.2870701551437378, "sampling/importance_sampling_ratio/max": 1.2086752653121948, "sampling/importance_sampling_ratio/mean": 0.5414099097251892, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.23090362548828, "sampling/sampling_logp_difference/mean": 2.138004779815674, "step": 367, "step_time": 18.158554118999746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00333847152069211, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00333847152069211, "completions/clipped_ratio": 0.0625, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 16.1875, "completions/mean_terminated_length": 15.666667938232422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.042856901884079, "epoch": 0.00736, "frac_reward_zero_std": 0.0, "grad_norm": 0.25630202889442444, "kl": 0.16829620581120253, "learning_rate": 7.588580352419923e-06, "loss": -0.1222, "num_tokens": 16490597.0, "reward": 0.10312500596046448, "reward_std": 0.2803059220314026, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.29782477021217346, "sampling/importance_sampling_ratio/max": 2.2433021068573, "sampling/importance_sampling_ratio/mean": 0.6510198712348938, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.54610061645508, "sampling/sampling_logp_difference/mean": 2.1730129718780518, "step": 368, "step_time": 18.308682317001512 }, { "clip_ratio/high_max": 0.005380036775022745, "clip_ratio/high_mean": 0.0026900183875113726, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003940018359571695, "completions/clipped_ratio": 0.09375, "completions/max_length": 29.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 15.875, "completions/mean_terminated_length": 15.758620262145996, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.1093397736549377, "epoch": 0.00738, "frac_reward_zero_std": 0.0, "grad_norm": 0.5375586152076721, "kl": 0.29412846895866096, "learning_rate": 7.575849168544717e-06, "loss": -0.036, "num_tokens": 16534290.0, "reward": 0.08437500894069672, "reward_std": 0.2218959927558899, "rewards/rollout_reward_func/mean": 0.08437500894069672, "rewards/rollout_reward_func/std": 0.25791895389556885, "sampling/importance_sampling_ratio/max": 1.4854854345321655, "sampling/importance_sampling_ratio/mean": 0.5502066612243652, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.69509506225586, "sampling/sampling_logp_difference/mean": 2.2163262367248535, "step": 369, "step_time": 18.90449294404243 }, { "clip_ratio/high_max": 0.009920635027810931, "clip_ratio/high_mean": 0.0063190130749717355, "clip_ratio/low_mean": 0.0059510982828214765, "clip_ratio/low_min": 0.0026041667442768812, "clip_ratio/region_mean": 0.012270111357793212, "completions/clipped_ratio": 0.09375, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 17.875, "completions/mean_terminated_length": 17.068965911865234, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1440256405621767, "epoch": 0.0074, "frac_reward_zero_std": 0.0, "grad_norm": 0.35702013969421387, "kl": 0.21060387254692614, "learning_rate": 7.5631004958227e-06, "loss": -0.1445, "num_tokens": 16578193.0, "reward": -0.0312499962747097, "reward_std": 0.34090518951416016, "rewards/rollout_reward_func/mean": -0.0312499962747097, "rewards/rollout_reward_func/std": 0.3559969961643219, "sampling/importance_sampling_ratio/max": 1.6255292892456055, "sampling/importance_sampling_ratio/mean": 0.5401814579963684, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.63715744018555, "sampling/sampling_logp_difference/mean": 2.5197958946228027, "step": 370, "step_time": 19.640131248001126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0021703839884139597, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021703839884139597, "completions/clipped_ratio": 0.1875, "completions/max_length": 77.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 16.59375, "completions/mean_terminated_length": 13.769230842590332, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9884212370961905, "epoch": 0.00742, "frac_reward_zero_std": 0.0, "grad_norm": 0.40004441142082214, "kl": 0.49209799501113594, "learning_rate": 7.550334502417483e-06, "loss": -0.0234, "num_tokens": 16622838.0, "reward": -0.02500000037252903, "reward_std": 0.2714792788028717, "rewards/rollout_reward_func/mean": -0.02500000037252903, "rewards/rollout_reward_func/std": 0.2782433331012726, "sampling/importance_sampling_ratio/max": 1.7687228918075562, "sampling/importance_sampling_ratio/mean": 0.5068832635879517, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.15277862548828, "sampling/sampling_logp_difference/mean": 1.995932698249817, "step": 371, "step_time": 19.390808668002137 }, { "clip_ratio/high_max": 0.01391117472667247, "clip_ratio/high_mean": 0.006955587363336235, "clip_ratio/low_mean": 0.008490720065310597, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015446307428646833, "completions/clipped_ratio": 0.15625, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 18.875, "completions/mean_terminated_length": 17.518518447875977, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2873956970870495, "epoch": 0.00744, "frac_reward_zero_std": 0.0, "grad_norm": 0.1875849813222885, "kl": 0.3253220268525183, "learning_rate": 7.537551356721149e-06, "loss": -0.0458, "num_tokens": 16668195.0, "reward": 0.012500002980232239, "reward_std": 0.31635284423828125, "rewards/rollout_reward_func/mean": 0.012500002980232239, "rewards/rollout_reward_func/std": 0.3405403792858124, "sampling/importance_sampling_ratio/max": 1.2712230682373047, "sampling/importance_sampling_ratio/mean": 0.35645580291748047, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.78174591064453, "sampling/sampling_logp_difference/mean": 2.8424582481384277, "step": 372, "step_time": 18.96847196800809 }, { "clip_ratio/high_max": 0.008399209706112742, "clip_ratio/high_mean": 0.004199604853056371, "clip_ratio/low_mean": 0.00703125010477379, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01123085495783016, "completions/clipped_ratio": 0.25, "completions/max_length": 505.0, "completions/max_terminated_length": 505.0, "completions/mean_length": 31.9375, "completions/mean_terminated_length": 35.458335876464844, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.124992173165083, "epoch": 0.00746, "frac_reward_zero_std": 0.0, "grad_norm": 0.3334440588951111, "kl": 0.18119482742622495, "learning_rate": 7.5247512273520325e-06, "loss": -0.0689, "num_tokens": 16713014.0, "reward": -0.03437499701976776, "reward_std": 0.3308691382408142, "rewards/rollout_reward_func/mean": -0.03437499701976776, "rewards/rollout_reward_func/std": 0.3441767990589142, "sampling/importance_sampling_ratio/max": 1.649154543876648, "sampling/importance_sampling_ratio/mean": 0.4804171919822693, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.607303619384766, "sampling/sampling_logp_difference/mean": 1.9895813465118408, "step": 373, "step_time": 22.40462595601275 }, { "clip_ratio/high_max": 0.0036764706019312143, "clip_ratio/high_mean": 0.0018382353009656072, "clip_ratio/low_mean": 0.0030691964784637094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0049074317794293165, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 14.59375, "completions/mean_terminated_length": 13.964286804199219, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.027576057240367, "epoch": 0.00748, "frac_reward_zero_std": 0.0, "grad_norm": 0.43433165550231934, "kl": 0.3038945463486016, "learning_rate": 7.511934283152491e-06, "loss": -0.1256, "num_tokens": 16757553.0, "reward": -0.006249997764825821, "reward_std": 0.2779816687107086, "rewards/rollout_reward_func/mean": -0.006249997764825821, "rewards/rollout_reward_func/std": 0.31205615401268005, "sampling/importance_sampling_ratio/max": 2.0503251552581787, "sampling/importance_sampling_ratio/mean": 0.5399911403656006, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.08975601196289, "sampling/sampling_logp_difference/mean": 2.2013978958129883, "step": 374, "step_time": 19.007944869983476 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011574074160307646, "completions/clipped_ratio": 0.125, "completions/max_length": 48.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 14.3125, "completions/mean_terminated_length": 12.928571701049805, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7905611582100391, "epoch": 0.0075, "frac_reward_zero_std": 0.0, "grad_norm": 0.3697602450847626, "kl": 0.25800791941583157, "learning_rate": 7.499100693186684e-06, "loss": -0.0364, "num_tokens": 16799196.0, "reward": 0.06562499701976776, "reward_std": 0.2855823338031769, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.32389700412750244, "sampling/importance_sampling_ratio/max": 1.8785368204116821, "sampling/importance_sampling_ratio/mean": 0.7126092910766602, "sampling/importance_sampling_ratio/min": 2.634441112930656e-43, "sampling/sampling_logp_difference/max": 33.774776458740234, "sampling/sampling_logp_difference/mean": 1.3636586666107178, "step": 375, "step_time": 17.97827509896888 }, { "clip_ratio/high_max": 0.021888438379392028, "clip_ratio/high_mean": 0.012680330197326839, "clip_ratio/low_mean": 0.0009191176504828036, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013599447847809643, "completions/clipped_ratio": 0.1875, "completions/max_length": 35.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 16.6875, "completions/mean_terminated_length": 16.730770111083984, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2929313499480486, "epoch": 0.00752, "frac_reward_zero_std": 0.0, "grad_norm": 0.24874383211135864, "kl": 0.16620376450009644, "learning_rate": 7.486250626738338e-06, "loss": -0.0325, "num_tokens": 16844578.0, "reward": 0.012500002980232239, "reward_std": 0.2775716185569763, "rewards/rollout_reward_func/mean": 0.012500002980232239, "rewards/rollout_reward_func/std": 0.3034745156764984, "sampling/importance_sampling_ratio/max": 1.5989837646484375, "sampling/importance_sampling_ratio/mean": 0.46749186515808105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.85135269165039, "sampling/sampling_logp_difference/mean": 2.8013744354248047, "step": 376, "step_time": 18.135622109024553 }, { "clip_ratio/high_max": 0.012467672349885106, "clip_ratio/high_mean": 0.006233836174942553, "clip_ratio/low_mean": 0.0055594645673409104, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011793300742283463, "completions/clipped_ratio": 0.21875, "completions/max_length": 51.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 18.8125, "completions/mean_terminated_length": 17.31999969482422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2092900164425373, "epoch": 0.00754, "frac_reward_zero_std": 0.0, "grad_norm": 0.4899267554283142, "kl": 0.2679423959925771, "learning_rate": 7.473384253308518e-06, "loss": -0.1257, "num_tokens": 16889658.0, "reward": 0.03437500447034836, "reward_std": 0.2920238971710205, "rewards/rollout_reward_func/mean": 0.03437500447034836, "rewards/rollout_reward_func/std": 0.3085966110229492, "sampling/importance_sampling_ratio/max": 1.2385331392288208, "sampling/importance_sampling_ratio/mean": 0.3543841242790222, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.76484298706055, "sampling/sampling_logp_difference/mean": 2.5530335903167725, "step": 377, "step_time": 19.42007513399585 }, { "clip_ratio/high_max": 0.0209179415833205, "clip_ratio/high_mean": 0.011879425379447639, "clip_ratio/low_mean": 0.0029761905898340046, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014855615969281644, "completions/clipped_ratio": 0.03125, "completions/max_length": 35.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 16.125, "completions/mean_terminated_length": 16.032257080078125, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3398886770009995, "epoch": 0.00756, "frac_reward_zero_std": 0.0, "grad_norm": 0.22456561028957367, "kl": 0.3633189285174012, "learning_rate": 7.460501742613385e-06, "loss": -0.0655, "num_tokens": 16934840.0, "reward": -0.012499997392296791, "reward_std": 0.3162919282913208, "rewards/rollout_reward_func/mean": -0.012499997392296791, "rewards/rollout_reward_func/std": 0.3108261823654175, "sampling/importance_sampling_ratio/max": 1.4097384214401245, "sampling/importance_sampling_ratio/mean": 0.33542564511299133, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.773799896240234, "sampling/sampling_logp_difference/mean": 3.011707305908203, "step": 378, "step_time": 18.230256314011058 }, { "clip_ratio/high_max": 0.026537699392065406, "clip_ratio/high_mean": 0.013268849696032703, "clip_ratio/low_mean": 0.003802910097874701, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017071759793907404, "completions/clipped_ratio": 0.21875, "completions/max_length": 27.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 15.875, "completions/mean_terminated_length": 14.59999942779541, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2023647148162127, "epoch": 0.00758, "frac_reward_zero_std": 0.0, "grad_norm": 0.48279526829719543, "kl": 0.7850466873496771, "learning_rate": 7.447603264581964e-06, "loss": -0.0596, "num_tokens": 16980110.0, "reward": 0.03437499701976776, "reward_std": 0.2791997790336609, "rewards/rollout_reward_func/mean": 0.03437499701976776, "rewards/rollout_reward_func/std": 0.2869296371936798, "sampling/importance_sampling_ratio/max": 2.75992488861084, "sampling/importance_sampling_ratio/mean": 0.41396504640579224, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.34850311279297, "sampling/sampling_logp_difference/mean": 2.4684228897094727, "step": 379, "step_time": 19.489168291023816 }, { "clip_ratio/high_max": 0.006623641354963183, "clip_ratio/high_mean": 0.0033118206774815917, "clip_ratio/low_mean": 0.004175967420451343, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007487788097932935, "completions/clipped_ratio": 0.125, "completions/max_length": 52.0, "completions/max_terminated_length": 52.0, "completions/mean_length": 18.4375, "completions/mean_terminated_length": 17.85714340209961, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2165225241333246, "epoch": 0.0076, "frac_reward_zero_std": 0.0, "grad_norm": 0.27314093708992004, "kl": 0.1302790038753301, "learning_rate": 7.4346889893539e-06, "loss": -0.1787, "num_tokens": 17025447.0, "reward": -0.015625, "reward_std": 0.3072485029697418, "rewards/rollout_reward_func/mean": -0.015625, "rewards/rollout_reward_func/std": 0.3341907560825348, "sampling/importance_sampling_ratio/max": 1.445472002029419, "sampling/importance_sampling_ratio/mean": 0.4045464098453522, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.294803619384766, "sampling/sampling_logp_difference/mean": 2.6115365028381348, "step": 380, "step_time": 19.37588474401855 }, { "clip_ratio/high_max": 0.022932331543415785, "clip_ratio/high_mean": 0.011466165771707892, "clip_ratio/low_mean": 0.00603938743006438, "clip_ratio/low_min": 0.0018382353009656072, "clip_ratio/region_mean": 0.017505553201772273, "completions/clipped_ratio": 0.1875, "completions/max_length": 34.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 18.4375, "completions/mean_terminated_length": 16.538461685180664, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2094893716275692, "epoch": 0.00762, "frac_reward_zero_std": 0.0, "grad_norm": 0.4589832127094269, "kl": 0.31773237884044647, "learning_rate": 7.421759087277214e-06, "loss": -0.102, "num_tokens": 17070924.0, "reward": -0.08749999105930328, "reward_std": 0.2816832661628723, "rewards/rollout_reward_func/mean": -0.08749999105930328, "rewards/rollout_reward_func/std": 0.2836968004703522, "sampling/importance_sampling_ratio/max": 2.324122905731201, "sampling/importance_sampling_ratio/mean": 0.3761470317840576, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.296077728271484, "sampling/sampling_logp_difference/mean": 2.455155372619629, "step": 381, "step_time": 20.37960985697282 }, { "clip_ratio/high_max": 0.00963446858804673, "clip_ratio/high_mean": 0.004817234294023365, "clip_ratio/low_mean": 0.00905257964041084, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013869813934434205, "completions/clipped_ratio": 0.09375, "completions/max_length": 35.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 17.65625, "completions/mean_terminated_length": 16.413793563842773, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2320054434239864, "epoch": 0.00764, "frac_reward_zero_std": 0.0, "grad_norm": 0.3147365152835846, "kl": 0.17552909534424543, "learning_rate": 7.408813728906053e-06, "loss": -0.0761, "num_tokens": 17116854.0, "reward": -0.05312499776482582, "reward_std": 0.26351410150527954, "rewards/rollout_reward_func/mean": -0.05312499776482582, "rewards/rollout_reward_func/std": 0.28282487392425537, "sampling/importance_sampling_ratio/max": 1.2994697093963623, "sampling/importance_sampling_ratio/mean": 0.32624995708465576, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.01215744018555, "sampling/sampling_logp_difference/mean": 2.988271474838257, "step": 382, "step_time": 19.01506717700977 }, { "clip_ratio/high_max": 0.007525083841755986, "clip_ratio/high_mean": 0.004964465158991516, "clip_ratio/low_mean": 0.006200397037900984, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0111648621968925, "completions/clipped_ratio": 0.09375, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 15.84375, "completions/mean_terminated_length": 15.275861740112305, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2239100597798824, "epoch": 0.00766, "frac_reward_zero_std": 0.0, "grad_norm": 0.22533005475997925, "kl": 0.1644873940385878, "learning_rate": 7.395853084998448e-06, "loss": -0.0286, "num_tokens": 17161741.0, "reward": 0.028125006705522537, "reward_std": 0.3513844609260559, "rewards/rollout_reward_func/mean": 0.028125006705522537, "rewards/rollout_reward_func/std": 0.3521678149700165, "sampling/importance_sampling_ratio/max": 1.2337796688079834, "sampling/importance_sampling_ratio/mean": 0.35786306858062744, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.23917007446289, "sampling/sampling_logp_difference/mean": 2.982215642929077, "step": 383, "step_time": 19.184433091024403 }, { "clip_ratio/high_max": 0.013736264314502478, "clip_ratio/high_mean": 0.006868132157251239, "clip_ratio/low_mean": 0.004978598328307271, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01184673048555851, "completions/clipped_ratio": 0.21875, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 17.71875, "completions/mean_terminated_length": 17.239999771118164, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3329155910760164, "epoch": 0.00768, "frac_reward_zero_std": 0.0, "grad_norm": 0.2625318467617035, "kl": 0.2077877086121589, "learning_rate": 7.382877326514051e-06, "loss": -0.0251, "num_tokens": 17207087.0, "reward": -0.025000009685754776, "reward_std": 0.3573606312274933, "rewards/rollout_reward_func/mean": -0.025000009685754776, "rewards/rollout_reward_func/std": 0.3733026087284088, "sampling/importance_sampling_ratio/max": 1.4276705980300903, "sampling/importance_sampling_ratio/mean": 0.3534102439880371, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.070098876953125, "sampling/sampling_logp_difference/mean": 2.8459689617156982, "step": 384, "step_time": 20.2835872379801 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0014880952658131719, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028467909432947636, "completions/clipped_ratio": 0.15625, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 16.96875, "completions/mean_terminated_length": 16.11111068725586, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0878392718732357, "epoch": 0.0077, "frac_reward_zero_std": 0.0, "grad_norm": 0.32289981842041016, "kl": 0.4796707849018276, "learning_rate": 7.369886624611889e-06, "loss": -0.1262, "num_tokens": 17251620.0, "reward": 0.08124999701976776, "reward_std": 0.30110710859298706, "rewards/rollout_reward_func/mean": 0.08124999701976776, "rewards/rollout_reward_func/std": 0.3156534433364868, "sampling/importance_sampling_ratio/max": 1.4836194515228271, "sampling/importance_sampling_ratio/mean": 0.41117966175079346, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.04608917236328, "sampling/sampling_logp_difference/mean": 2.127199649810791, "step": 385, "step_time": 18.735521691982285 }, { "clip_ratio/high_max": 0.009037612937390804, "clip_ratio/high_mean": 0.004518806468695402, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005676213884726167, "completions/clipped_ratio": 0.15625, "completions/max_length": 29.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 13.3125, "completions/mean_terminated_length": 12.407407760620117, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.8912513218820095, "epoch": 0.00772, "frac_reward_zero_std": 0.0, "grad_norm": 0.4255545735359192, "kl": 0.4221990266814828, "learning_rate": 7.356881150648103e-06, "loss": -0.0421, "num_tokens": 17294884.0, "reward": 0.05625000223517418, "reward_std": 0.3279723525047302, "rewards/rollout_reward_func/mean": 0.05625000223517418, "rewards/rollout_reward_func/std": 0.31718266010284424, "sampling/importance_sampling_ratio/max": 2.110640287399292, "sampling/importance_sampling_ratio/mean": 0.616936445236206, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.25479507446289, "sampling/sampling_logp_difference/mean": 1.7001668214797974, "step": 386, "step_time": 17.761482151021482 }, { "clip_ratio/high_max": 0.018342391354963183, "clip_ratio/high_mean": 0.010815932764671743, "clip_ratio/low_mean": 0.0020833334419876337, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012899266206659377, "completions/clipped_ratio": 0.09375, "completions/max_length": 30.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 14.03125, "completions/mean_terminated_length": 12.758620262145996, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.172169093042612, "epoch": 0.00774, "frac_reward_zero_std": 0.0, "grad_norm": 0.2343388944864273, "kl": 0.44868631567806005, "learning_rate": 7.343861076173684e-06, "loss": -0.0123, "num_tokens": 17339238.0, "reward": 0.02812499925494194, "reward_std": 0.29020315408706665, "rewards/rollout_reward_func/mean": 0.02812499925494194, "rewards/rollout_reward_func/std": 0.30715590715408325, "sampling/importance_sampling_ratio/max": 1.6229959726333618, "sampling/importance_sampling_ratio/mean": 0.45830732583999634, "sampling/importance_sampling_ratio/min": 5.74532370373175e-44, "sampling/sampling_logp_difference/max": 33.5942497253418, "sampling/sampling_logp_difference/mean": 2.8695755004882812, "step": 387, "step_time": 18.56014032100211 }, { "clip_ratio/high_max": 0.026123272720724344, "clip_ratio/high_mean": 0.013061636360362172, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013061636360362172, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 15.5625, "completions/mean_terminated_length": 12.639999389648438, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0597389936447144, "epoch": 0.00776, "frac_reward_zero_std": 0.0, "grad_norm": 0.3771252930164337, "kl": 0.44486150378361344, "learning_rate": 7.330826572932217e-06, "loss": -0.0977, "num_tokens": 17385535.0, "reward": -0.08437500149011612, "reward_std": 0.3877263069152832, "rewards/rollout_reward_func/mean": -0.08437500149011612, "rewards/rollout_reward_func/std": 0.376836895942688, "sampling/importance_sampling_ratio/max": 1.1617588996887207, "sampling/importance_sampling_ratio/mean": 0.3429880142211914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.92107391357422, "sampling/sampling_logp_difference/mean": 2.119088649749756, "step": 388, "step_time": 18.66556587202649 }, { "clip_ratio/high_max": 0.023276601918041706, "clip_ratio/high_mean": 0.011638300959020853, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012996996636502445, "completions/clipped_ratio": 0.25, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 20.34375, "completions/mean_terminated_length": 18.166667938232422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3286371007561684, "epoch": 0.00778, "frac_reward_zero_std": 0.0, "grad_norm": 0.18569201231002808, "kl": 0.1448280750773847, "learning_rate": 7.3177778128576124e-06, "loss": -0.0667, "num_tokens": 17431953.0, "reward": -0.03125, "reward_std": 0.3403507173061371, "rewards/rollout_reward_func/mean": -0.03125, "rewards/rollout_reward_func/std": 0.3325730562210083, "sampling/importance_sampling_ratio/max": 1.3558939695358276, "sampling/importance_sampling_ratio/mean": 0.3371841311454773, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.69135665893555, "sampling/sampling_logp_difference/mean": 2.6602530479431152, "step": 389, "step_time": 20.359560829994734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 17.9375, "completions/mean_terminated_length": 16.239999771118164, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9629128500819206, "epoch": 0.0078, "frac_reward_zero_std": 0.0, "grad_norm": 0.3996964991092682, "kl": 0.23106009827461094, "learning_rate": 7.304714968071834e-06, "loss": -0.1361, "num_tokens": 17475924.0, "reward": -0.01875000074505806, "reward_std": 0.39310139417648315, "rewards/rollout_reward_func/mean": -0.01875000074505806, "rewards/rollout_reward_func/std": 0.38557666540145874, "sampling/importance_sampling_ratio/max": 1.5701305866241455, "sampling/importance_sampling_ratio/mean": 0.48899149894714355, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.884151458740234, "sampling/sampling_logp_difference/mean": 1.7728092670440674, "step": 390, "step_time": 19.05107201600913 }, { "clip_ratio/high_max": 0.005580357275903225, "clip_ratio/high_mean": 0.0027901786379516125, "clip_ratio/low_mean": 0.006175723858177662, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008965902379713953, "completions/clipped_ratio": 0.21875, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 19.6875, "completions/mean_terminated_length": 18.520000457763672, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1961617469787598, "epoch": 0.00782, "frac_reward_zero_std": 0.0, "grad_norm": 0.17372283339500427, "kl": 0.22758979978971183, "learning_rate": 7.291638210882638e-06, "loss": -0.0957, "num_tokens": 17520800.0, "reward": 0.08750000596046448, "reward_std": 0.3172113001346588, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.3076969385147095, "sampling/importance_sampling_ratio/max": 1.5343787670135498, "sampling/importance_sampling_ratio/mean": 0.4082627296447754, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.281768798828125, "sampling/sampling_logp_difference/mean": 2.541497230529785, "step": 391, "step_time": 19.63212031200237 }, { "clip_ratio/high_max": 0.014610390178859234, "clip_ratio/high_mean": 0.007305195089429617, "clip_ratio/low_mean": 0.0038768798112869263, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011182074900716543, "completions/clipped_ratio": 0.03125, "completions/max_length": 26.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 12.25, "completions/mean_terminated_length": 12.032258033752441, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9848070973530412, "epoch": 0.00784, "frac_reward_zero_std": 0.0, "grad_norm": 0.45328694581985474, "kl": 1.9072131924331188, "learning_rate": 7.278547713781288e-06, "loss": 0.0104, "num_tokens": 17565329.0, "reward": 0.04374999925494194, "reward_std": 0.31722909212112427, "rewards/rollout_reward_func/mean": 0.04374999925494194, "rewards/rollout_reward_func/std": 0.31102070212364197, "sampling/importance_sampling_ratio/max": 1.3768523931503296, "sampling/importance_sampling_ratio/mean": 0.5786175727844238, "sampling/importance_sampling_ratio/min": 1.9057659114817512e-43, "sampling/sampling_logp_difference/max": 33.35858917236328, "sampling/sampling_logp_difference/mean": 2.405778169631958, "step": 392, "step_time": 18.216178717979346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016447368543595076, "completions/clipped_ratio": 0.125, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 15.0625, "completions/mean_terminated_length": 14.500000953674316, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.959168391302228, "epoch": 0.00786, "frac_reward_zero_std": 0.0, "grad_norm": 0.2736896574497223, "kl": 0.16285047074779868, "learning_rate": 7.2654436494402955e-06, "loss": -0.1405, "num_tokens": 17608865.0, "reward": 0.046875, "reward_std": 0.32068154215812683, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.32724103331565857, "sampling/importance_sampling_ratio/max": 1.3174887895584106, "sampling/importance_sampling_ratio/mean": 0.575310230255127, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.7480583190918, "sampling/sampling_logp_difference/mean": 2.1490681171417236, "step": 393, "step_time": 18.250532356993062 }, { "clip_ratio/high_max": 0.006189613603055477, "clip_ratio/high_mean": 0.0030948068015277386, "clip_ratio/low_mean": 0.0026900183875113726, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005784825189039111, "completions/clipped_ratio": 0.15625, "completions/max_length": 26.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 14.21875, "completions/mean_terminated_length": 13.777777671813965, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0031289476901293, "epoch": 0.00788, "frac_reward_zero_std": 0.0, "grad_norm": 0.3889482319355011, "kl": 0.3244193266145885, "learning_rate": 7.252326190711121e-06, "loss": -0.1667, "num_tokens": 17652286.0, "reward": 0.09375, "reward_std": 0.2560943365097046, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.26873359084129333, "sampling/importance_sampling_ratio/max": 2.199491024017334, "sampling/importance_sampling_ratio/mean": 0.5993974804878235, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 33.1487922668457, "sampling/sampling_logp_difference/mean": 2.2160582542419434, "step": 394, "step_time": 18.394460798954242 }, { "clip_ratio/high_max": 0.005217391299083829, "clip_ratio/high_mean": 0.0026086956495419145, "clip_ratio/low_mean": 0.004048583097755909, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006657278747297823, "completions/clipped_ratio": 0.1875, "completions/max_length": 35.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.3125, "completions/mean_terminated_length": 15.769231796264648, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2043715212494135, "epoch": 0.0079, "frac_reward_zero_std": 0.0, "grad_norm": 0.20503124594688416, "kl": 0.1437356334645301, "learning_rate": 7.239195510621918e-06, "loss": -0.1231, "num_tokens": 17697279.0, "reward": 0.028125004842877388, "reward_std": 0.34498143196105957, "rewards/rollout_reward_func/mean": 0.028125004842877388, "rewards/rollout_reward_func/std": 0.3343113958835602, "sampling/importance_sampling_ratio/max": 1.365868091583252, "sampling/importance_sampling_ratio/mean": 0.4518762230873108, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.875518798828125, "sampling/sampling_logp_difference/mean": 2.411149024963379, "step": 395, "step_time": 18.730498961987905 }, { "clip_ratio/high_max": 0.011431333841755986, "clip_ratio/high_mean": 0.005715666920877993, "clip_ratio/low_mean": 0.0011160714784637094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0068317383993417025, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 13.90625, "completions/mean_terminated_length": 11.185185432434082, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2052724687382579, "epoch": 0.00792, "frac_reward_zero_std": 0.0, "grad_norm": 0.22880469262599945, "kl": 0.4353255182504654, "learning_rate": 7.22605178237523e-06, "loss": -0.0638, "num_tokens": 17741944.0, "reward": 0.04062500596046448, "reward_std": 0.31669726967811584, "rewards/rollout_reward_func/mean": 0.04062500596046448, "rewards/rollout_reward_func/std": 0.35637909173965454, "sampling/importance_sampling_ratio/max": 1.3920466899871826, "sampling/importance_sampling_ratio/mean": 0.44314301013946533, "sampling/importance_sampling_ratio/min": 2.4242463432819335e-43, "sampling/sampling_logp_difference/max": 33.38984298706055, "sampling/sampling_logp_difference/mean": 2.6771602630615234, "step": 396, "step_time": 18.93803414599097 }, { "clip_ratio/high_max": 0.005340909119695425, "clip_ratio/high_mean": 0.0038723775651305914, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00610452052205801, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 15.8125, "completions/mean_terminated_length": 14.666666984558105, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.1983209550380707, "epoch": 0.00794, "frac_reward_zero_std": 0.0, "grad_norm": 0.22504577040672302, "kl": 0.33653818955644965, "learning_rate": 7.212895179345718e-06, "loss": -0.1201, "num_tokens": 17786456.0, "reward": 0.02812499925494194, "reward_std": 0.3453521728515625, "rewards/rollout_reward_func/mean": 0.02812499925494194, "rewards/rollout_reward_func/std": 0.3381490111351013, "sampling/importance_sampling_ratio/max": 1.1307518482208252, "sampling/importance_sampling_ratio/mean": 0.3178793787956238, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.60859298706055, "sampling/sampling_logp_difference/mean": 2.757720947265625, "step": 397, "step_time": 18.466383295992273 }, { "clip_ratio/high_max": 0.016079696360975504, "clip_ratio/high_mean": 0.008039848180487752, "clip_ratio/low_mean": 0.0071106323739513755, "clip_ratio/low_min": 0.002314814832061529, "clip_ratio/region_mean": 0.015150480554439127, "completions/clipped_ratio": 0.3125, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 19.625, "completions/mean_terminated_length": 18.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3490084912627935, "epoch": 0.00796, "frac_reward_zero_std": 0.0, "grad_norm": 0.24709999561309814, "kl": 0.2978859612485394, "learning_rate": 7.199725875077871e-06, "loss": -0.1114, "num_tokens": 17832267.0, "reward": -0.10312499850988388, "reward_std": 0.3225356340408325, "rewards/rollout_reward_func/mean": -0.10312499850988388, "rewards/rollout_reward_func/std": 0.32277464866638184, "sampling/importance_sampling_ratio/max": 1.3353843688964844, "sampling/importance_sampling_ratio/mean": 0.3443281054496765, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.711280822753906, "sampling/sampling_logp_difference/mean": 2.637747287750244, "step": 398, "step_time": 19.534635550968233 }, { "clip_ratio/high_max": 0.008956643054261804, "clip_ratio/high_mean": 0.004478321527130902, "clip_ratio/low_mean": 0.004582771100103855, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009061092627234757, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 16.3125, "completions/mean_terminated_length": 15.481481552124023, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2169874422252178, "epoch": 0.00798, "frac_reward_zero_std": 0.0, "grad_norm": 0.2878795266151428, "kl": 1.2247957191430032, "learning_rate": 7.186544043283715e-06, "loss": -0.0567, "num_tokens": 17876635.0, "reward": -0.02187499962747097, "reward_std": 0.28095364570617676, "rewards/rollout_reward_func/mean": -0.02187499962747097, "rewards/rollout_reward_func/std": 0.2802468538284302, "sampling/importance_sampling_ratio/max": 1.2393170595169067, "sampling/importance_sampling_ratio/mean": 0.4005240201950073, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 33.28602981567383, "sampling/sampling_logp_difference/mean": 2.410452365875244, "step": 399, "step_time": 18.72510329101351 }, { "clip_ratio/high_max": 0.007573052076622844, "clip_ratio/high_mean": 0.003786526038311422, "clip_ratio/low_mean": 0.0055509868543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00933751289267093, "completions/clipped_ratio": 0.15625, "completions/max_length": 35.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 17.125, "completions/mean_terminated_length": 16.370370864868164, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1867173574864864, "epoch": 0.008, "frac_reward_zero_std": 0.0, "grad_norm": 0.30171117186546326, "kl": 0.14222361729480326, "learning_rate": 7.173349857840521e-06, "loss": -0.0557, "num_tokens": 17921584.0, "reward": 0.04062500596046448, "reward_std": 0.33055445551872253, "rewards/rollout_reward_func/mean": 0.04062500596046448, "rewards/rollout_reward_func/std": 0.32711777091026306, "sampling/importance_sampling_ratio/max": 1.4536683559417725, "sampling/importance_sampling_ratio/mean": 0.49011850357055664, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.69218826293945, "sampling/sampling_logp_difference/mean": 2.3511621952056885, "step": 400, "step_time": 18.18722395999066 }, { "clip_ratio/high_max": 0.003289473708719015, "clip_ratio/high_mean": 0.0016447368543595076, "clip_ratio/low_mean": 0.0072256840649060905, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008870420919265598, "completions/clipped_ratio": 0.21875, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 16.6875, "completions/mean_terminated_length": 15.079999923706055, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2628629133105278, "epoch": 0.00802, "frac_reward_zero_std": 0.0, "grad_norm": 0.5107921361923218, "kl": 0.2593350429087877, "learning_rate": 7.160143492788516e-06, "loss": -0.0881, "num_tokens": 17966849.0, "reward": 0.006250001490116119, "reward_std": 0.2959166169166565, "rewards/rollout_reward_func/mean": 0.006250001490116119, "rewards/rollout_reward_func/std": 0.3262074291706085, "sampling/importance_sampling_ratio/max": 2.066516637802124, "sampling/importance_sampling_ratio/mean": 0.4556736350059509, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.08881759643555, "sampling/sampling_logp_difference/mean": 2.468008518218994, "step": 401, "step_time": 19.32202960694849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0034420291194692254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034420291194692254, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 13.90625, "completions/mean_terminated_length": 12.928571701049805, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.9821083825081587, "epoch": 0.00804, "frac_reward_zero_std": 0.0, "grad_norm": 0.7926605939865112, "kl": 0.3257042081095278, "learning_rate": 7.146925122328581e-06, "loss": -0.1391, "num_tokens": 18010621.0, "reward": 0.078125, "reward_std": 0.28016895055770874, "rewards/rollout_reward_func/mean": 0.078125, "rewards/rollout_reward_func/std": 0.28819164633750916, "sampling/importance_sampling_ratio/max": 2.001193046569824, "sampling/importance_sampling_ratio/mean": 0.6017515063285828, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.358585357666016, "sampling/sampling_logp_difference/mean": 2.050752878189087, "step": 402, "step_time": 17.283287623009528 }, { "clip_ratio/high_max": 0.011904762359336019, "clip_ratio/high_mean": 0.005952381179668009, "clip_ratio/low_mean": 0.003441220265813172, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009393601445481181, "completions/clipped_ratio": 0.125, "completions/max_length": 48.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.5, "completions/mean_terminated_length": 15.464286804199219, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1252037435770035, "epoch": 0.00806, "frac_reward_zero_std": 0.0, "grad_norm": 0.2831166088581085, "kl": 0.15065983356907964, "learning_rate": 7.133694920819958e-06, "loss": -0.1151, "num_tokens": 18056063.0, "reward": -0.03125, "reward_std": 0.3251188397407532, "rewards/rollout_reward_func/mean": -0.03125, "rewards/rollout_reward_func/std": 0.33929479122161865, "sampling/importance_sampling_ratio/max": 1.2449452877044678, "sampling/importance_sampling_ratio/mean": 0.3657669425010681, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.07733917236328, "sampling/sampling_logp_difference/mean": 2.126800537109375, "step": 403, "step_time": 19.492066000966588 }, { "clip_ratio/high_max": 0.02449260512366891, "clip_ratio/high_mean": 0.012246302561834455, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012246302561834455, "completions/clipped_ratio": 0.1875, "completions/max_length": 46.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.46875, "completions/mean_terminated_length": 17.5, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.1830239202827215, "epoch": 0.00808, "frac_reward_zero_std": 0.0, "grad_norm": 0.26731884479522705, "kl": 0.2231564656831324, "learning_rate": 7.12045306277795e-06, "loss": -0.1464, "num_tokens": 18101267.0, "reward": -0.021874994039535522, "reward_std": 0.37629956007003784, "rewards/rollout_reward_func/mean": -0.021874994039535522, "rewards/rollout_reward_func/std": 0.3899829387664795, "sampling/importance_sampling_ratio/max": 1.2140620946884155, "sampling/importance_sampling_ratio/mean": 0.33817851543426514, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.4355583190918, "sampling/sampling_logp_difference/mean": 2.3175785541534424, "step": 404, "step_time": 19.385628670977894 }, { "clip_ratio/high_max": 0.0077173912432044744, "clip_ratio/high_mean": 0.0038586956216022372, "clip_ratio/low_mean": 0.0048926768358796835, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00875137245748192, "completions/clipped_ratio": 0.25, "completions/max_length": 44.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 16.34375, "completions/mean_terminated_length": 13.833333969116211, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.192787142470479, "epoch": 0.0081, "frac_reward_zero_std": 0.0, "grad_norm": 0.4274740219116211, "kl": 0.46349997422657907, "learning_rate": 7.107199722871614e-06, "loss": -0.1878, "num_tokens": 18145587.0, "reward": 0.02812500298023224, "reward_std": 0.29584354162216187, "rewards/rollout_reward_func/mean": 0.02812500298023224, "rewards/rollout_reward_func/std": 0.3113284707069397, "sampling/importance_sampling_ratio/max": 1.5401504039764404, "sampling/importance_sampling_ratio/mean": 0.48976245522499084, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.79340744018555, "sampling/sampling_logp_difference/mean": 2.6256136894226074, "step": 405, "step_time": 18.550220859018737 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.004949534311890602, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006308229989372194, "completions/clipped_ratio": 0.1875, "completions/max_length": 44.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 17.78125, "completions/mean_terminated_length": 16.038461685180664, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2395285815000534, "epoch": 0.00812, "frac_reward_zero_std": 0.0, "grad_norm": 0.3571900725364685, "kl": 0.5302704207133502, "learning_rate": 7.093935075921465e-06, "loss": -0.0992, "num_tokens": 18191009.0, "reward": -0.08437500149011612, "reward_std": 0.3566862940788269, "rewards/rollout_reward_func/mean": -0.08437500149011612, "rewards/rollout_reward_func/std": 0.3584098815917969, "sampling/importance_sampling_ratio/max": 2.141678810119629, "sampling/importance_sampling_ratio/mean": 0.322803258895874, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.733585357666016, "sampling/sampling_logp_difference/mean": 2.5279006958007812, "step": 406, "step_time": 19.906919455024763 }, { "clip_ratio/high_max": 0.028882576851174235, "clip_ratio/high_mean": 0.014441288425587118, "clip_ratio/low_mean": 0.0026086956495419145, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017049984075129032, "completions/clipped_ratio": 0.15625, "completions/max_length": 47.0, "completions/max_terminated_length": 47.0, "completions/mean_length": 17.5, "completions/mean_terminated_length": 17.629629135131836, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.28056326135993, "epoch": 0.00814, "frac_reward_zero_std": 0.0, "grad_norm": 0.2844206392765045, "kl": 0.28792992141097784, "learning_rate": 7.0806592968971624e-06, "loss": -0.0992, "num_tokens": 18235957.0, "reward": -0.11875000596046448, "reward_std": 0.34642720222473145, "rewards/rollout_reward_func/mean": -0.11875000596046448, "rewards/rollout_reward_func/std": 0.3550897240638733, "sampling/importance_sampling_ratio/max": 1.3295248746871948, "sampling/importance_sampling_ratio/mean": 0.3479725122451782, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.1497802734375, "sampling/sampling_logp_difference/mean": 2.513529062271118, "step": 407, "step_time": 18.647688406010275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0022265624720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022265624720603228, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 15.0, "completions/mean_terminated_length": 13.199999809265137, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0759303914383054, "epoch": 0.00816, "frac_reward_zero_std": 0.0, "grad_norm": 0.3136056065559387, "kl": 0.3834100919775665, "learning_rate": 7.067372560915205e-06, "loss": -0.0748, "num_tokens": 18279687.0, "reward": 0.015625, "reward_std": 0.2943190038204193, "rewards/rollout_reward_func/mean": 0.015625, "rewards/rollout_reward_func/std": 0.30702459812164307, "sampling/importance_sampling_ratio/max": 2.43621563911438, "sampling/importance_sampling_ratio/mean": 0.5943390130996704, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.07465744018555, "sampling/sampling_logp_difference/mean": 1.97837495803833, "step": 408, "step_time": 18.10756645697984 }, { "clip_ratio/high_max": 0.011050347238779068, "clip_ratio/high_mean": 0.005525173619389534, "clip_ratio/low_mean": 0.0030598959419876337, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008585069561377168, "completions/clipped_ratio": 0.09375, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 18.40625, "completions/mean_terminated_length": 18.55172348022461, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.3393550254404545, "epoch": 0.00818, "frac_reward_zero_std": 0.0, "grad_norm": 0.3000224530696869, "kl": 0.1957079297862947, "learning_rate": 7.054075043236623e-06, "loss": -0.0832, "num_tokens": 18324636.0, "reward": 0.04062499850988388, "reward_std": 0.3588637709617615, "rewards/rollout_reward_func/mean": 0.04062499850988388, "rewards/rollout_reward_func/std": 0.3425326645374298, "sampling/importance_sampling_ratio/max": 1.20639169216156, "sampling/importance_sampling_ratio/mean": 0.3774069547653198, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.9110107421875, "sampling/sampling_logp_difference/mean": 2.661829710006714, "step": 409, "step_time": 18.839877313017496 }, { "clip_ratio/high_max": 0.010074429446831346, "clip_ratio/high_mean": 0.005037214723415673, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005037214723415673, "completions/clipped_ratio": 0.21875, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 18.34375, "completions/mean_terminated_length": 17.479999542236328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1025576498359442, "epoch": 0.0082, "frac_reward_zero_std": 0.0, "grad_norm": 0.3402940630912781, "kl": 0.12718838173896074, "learning_rate": 7.040766919264664e-06, "loss": -0.1199, "num_tokens": 18369455.0, "reward": 0.01562500186264515, "reward_std": 0.375651478767395, "rewards/rollout_reward_func/mean": 0.01562500186264515, "rewards/rollout_reward_func/std": 0.37769195437431335, "sampling/importance_sampling_ratio/max": 1.311021089553833, "sampling/importance_sampling_ratio/mean": 0.5020754933357239, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.523799896240234, "sampling/sampling_logp_difference/mean": 2.289320945739746, "step": 410, "step_time": 18.61467209800321 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.006262400886043906, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007621096563525498, "completions/clipped_ratio": 0.09375, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 18.9375, "completions/mean_terminated_length": 18.689655303955078, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2230697125196457, "epoch": 0.00822, "frac_reward_zero_std": 0.0, "grad_norm": 0.24137665331363678, "kl": 0.20833220309577882, "learning_rate": 7.027448364542479e-06, "loss": -0.1378, "num_tokens": 18414482.0, "reward": 0.012500006705522537, "reward_std": 0.2773970365524292, "rewards/rollout_reward_func/mean": 0.012500006705522537, "rewards/rollout_reward_func/std": 0.31289491057395935, "sampling/importance_sampling_ratio/max": 2.649320125579834, "sampling/importance_sampling_ratio/mean": 0.4296870529651642, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.38889694213867, "sampling/sampling_logp_difference/mean": 2.856074571609497, "step": 411, "step_time": 19.693780828019953 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0038879665080457926, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005246662185527384, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.84375, "completions/mean_terminated_length": 20.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1551939006894827, "epoch": 0.00824, "frac_reward_zero_std": 0.0, "grad_norm": 0.4079853296279907, "kl": 0.19788177264854312, "learning_rate": 7.01411955475081e-06, "loss": -0.1173, "num_tokens": 18460083.0, "reward": 0.03125, "reward_std": 0.2695818841457367, "rewards/rollout_reward_func/mean": 0.03125, "rewards/rollout_reward_func/std": 0.29009175300598145, "sampling/importance_sampling_ratio/max": 1.788041353225708, "sampling/importance_sampling_ratio/mean": 0.4004868268966675, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.94633102416992, "sampling/sampling_logp_difference/mean": 2.2658863067626953, "step": 412, "step_time": 18.887282138995943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.005242598708719015, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005242598708719015, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.15625, "completions/mean_terminated_length": 15.519999504089355, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0374756455421448, "epoch": 0.00826, "frac_reward_zero_std": 0.0, "grad_norm": 0.3774731755256653, "kl": 0.41878959082532674, "learning_rate": 7.000780665705665e-06, "loss": -0.1654, "num_tokens": 18503856.0, "reward": 0.0031250063329935074, "reward_std": 0.33140838146209717, "rewards/rollout_reward_func/mean": 0.0031250063329935074, "rewards/rollout_reward_func/std": 0.3345525562763214, "sampling/importance_sampling_ratio/max": 1.326322317123413, "sampling/importance_sampling_ratio/mean": 0.5454327464103699, "sampling/importance_sampling_ratio/min": 2.3962203739954372e-42, "sampling/sampling_logp_difference/max": 33.4925422668457, "sampling/sampling_logp_difference/mean": 2.0641040802001953, "step": 413, "step_time": 19.388660362019436 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0027901786379516125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004148874315433204, "completions/clipped_ratio": 0.1875, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 19.8125, "completions/mean_terminated_length": 18.538461685180664, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.134009562432766, "epoch": 0.00828, "frac_reward_zero_std": 0.0, "grad_norm": 0.28451889753341675, "kl": 0.31434807390905917, "learning_rate": 6.987431873356011e-06, "loss": -0.1768, "num_tokens": 18548623.0, "reward": -0.08437499403953552, "reward_std": 0.34728968143463135, "rewards/rollout_reward_func/mean": -0.08437499403953552, "rewards/rollout_reward_func/std": 0.3370741009712219, "sampling/importance_sampling_ratio/max": 2.964035749435425, "sampling/importance_sampling_ratio/mean": 0.4109351634979248, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 35.10136032104492, "sampling/sampling_logp_difference/mean": 2.428292989730835, "step": 414, "step_time": 19.726910354977008 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.004389406880363822, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005809861468151212, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.53125, "completions/mean_terminated_length": 16.68000030517578, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0830908417701721, "epoch": 0.0083, "frac_reward_zero_std": 0.0, "grad_norm": 0.3537006676197052, "kl": 0.22942752134986222, "learning_rate": 6.97407335378144e-06, "loss": -0.0815, "num_tokens": 18593207.0, "reward": -0.05625000223517418, "reward_std": 0.36796054244041443, "rewards/rollout_reward_func/mean": -0.05625000223517418, "rewards/rollout_reward_func/std": 0.3627293109893799, "sampling/importance_sampling_ratio/max": 2.240107774734497, "sampling/importance_sampling_ratio/mean": 0.41388195753097534, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.75759506225586, "sampling/sampling_logp_difference/mean": 1.934747576713562, "step": 415, "step_time": 19.792245804987033 }, { "clip_ratio/high_max": 0.010423629777505994, "clip_ratio/high_mean": 0.005211814888752997, "clip_ratio/low_mean": 0.006418701552320272, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011630516441073269, "completions/clipped_ratio": 0.1875, "completions/max_length": 42.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 21.09375, "completions/mean_terminated_length": 19.5, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1317945923656225, "epoch": 0.00832, "frac_reward_zero_std": 0.0, "grad_norm": 0.3256860375404358, "kl": 0.20869610272347927, "learning_rate": 6.960705283189857e-06, "loss": -0.1515, "num_tokens": 18638779.0, "reward": -0.04062499850988388, "reward_std": 0.3400871455669403, "rewards/rollout_reward_func/mean": -0.04062499850988388, "rewards/rollout_reward_func/std": 0.3536531925201416, "sampling/importance_sampling_ratio/max": 1.9280518293380737, "sampling/importance_sampling_ratio/mean": 0.3880004286766052, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.61851501464844, "sampling/sampling_logp_difference/mean": 2.478459358215332, "step": 416, "step_time": 19.603977948980173 }, { "clip_ratio/high_max": 0.006189613603055477, "clip_ratio/high_mean": 0.0030948068015277386, "clip_ratio/low_mean": 0.002379669575020671, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0054744763765484095, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 17.625, "completions/mean_terminated_length": 17.38461685180664, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.151793871074915, "epoch": 0.00834, "frac_reward_zero_std": 0.0, "grad_norm": 0.31688886880874634, "kl": 0.20592148741707206, "learning_rate": 6.94732783791515e-06, "loss": -0.2018, "num_tokens": 18684178.0, "reward": 0.02499999850988388, "reward_std": 0.25771939754486084, "rewards/rollout_reward_func/mean": 0.02499999850988388, "rewards/rollout_reward_func/std": 0.2805524468421936, "sampling/importance_sampling_ratio/max": 2.1858296394348145, "sampling/importance_sampling_ratio/mean": 0.4181938171386719, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 35.47636032104492, "sampling/sampling_logp_difference/mean": 2.3216922283172607, "step": 417, "step_time": 18.636823963039205 }, { "clip_ratio/high_max": 0.004807692486792803, "clip_ratio/high_mean": 0.0024038462433964014, "clip_ratio/low_mean": 0.0007102272938936949, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031140735372900963, "completions/clipped_ratio": 0.125, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 18.34375, "completions/mean_terminated_length": 17.678571701049805, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9602168500423431, "epoch": 0.00836, "frac_reward_zero_std": 0.0, "grad_norm": 0.39493295550346375, "kl": 0.16418810631148517, "learning_rate": 6.933941194414866e-06, "loss": -0.1308, "num_tokens": 18728742.0, "reward": 0.0625000074505806, "reward_std": 0.2559018135070801, "rewards/rollout_reward_func/mean": 0.0625000074505806, "rewards/rollout_reward_func/std": 0.2720887064933777, "sampling/importance_sampling_ratio/max": 1.4007086753845215, "sampling/importance_sampling_ratio/mean": 0.4697217345237732, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.188018798828125, "sampling/sampling_logp_difference/mean": 2.2332839965820312, "step": 418, "step_time": 18.927934677019948 }, { "clip_ratio/high_max": 0.01129380869679153, "clip_ratio/high_mean": 0.005646904348395765, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0070056000258773565, "completions/clipped_ratio": 0.21875, "completions/max_length": 31.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.5625, "completions/mean_terminated_length": 17.279998779296875, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1009340323507786, "epoch": 0.00838, "frac_reward_zero_std": 0.0, "grad_norm": 0.31180188059806824, "kl": 0.2279386413283646, "learning_rate": 6.920545529267882e-06, "loss": -0.0901, "num_tokens": 18772885.0, "reward": 0.109375, "reward_std": 0.26488617062568665, "rewards/rollout_reward_func/mean": 0.109375, "rewards/rollout_reward_func/std": 0.2644036114215851, "sampling/importance_sampling_ratio/max": 1.5458046197891235, "sampling/importance_sampling_ratio/mean": 0.49990415573120117, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.42014694213867, "sampling/sampling_logp_difference/mean": 2.260152816772461, "step": 419, "step_time": 18.596462321002036 }, { "clip_ratio/high_max": 0.006761695956811309, "clip_ratio/high_mean": 0.0033808479784056544, "clip_ratio/low_mean": 0.002003205183427781, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005384053161833435, "completions/clipped_ratio": 0.25, "completions/max_length": 49.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 21.6875, "completions/mean_terminated_length": 20.75, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.2401982098817825, "epoch": 0.0084, "frac_reward_zero_std": 0.0, "grad_norm": 0.3854202926158905, "kl": 0.20292459172196686, "learning_rate": 6.907141019172076e-06, "loss": -0.1143, "num_tokens": 18819056.0, "reward": -0.140625, "reward_std": 0.3252826929092407, "rewards/rollout_reward_func/mean": -0.140625, "rewards/rollout_reward_func/std": 0.3251395523548126, "sampling/importance_sampling_ratio/max": 1.6013134717941284, "sampling/importance_sampling_ratio/mean": 0.3775879442691803, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.64448928833008, "sampling/sampling_logp_difference/mean": 2.2048583030700684, "step": 420, "step_time": 20.61164483099128 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.002003205183427781, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034913004492409527, "completions/clipped_ratio": 0.15625, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 19.125, "completions/mean_terminated_length": 17.44444465637207, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9703402854502201, "epoch": 0.00842, "frac_reward_zero_std": 0.0, "grad_norm": 0.46120205521583557, "kl": 0.21708852075971663, "learning_rate": 6.893727840941997e-06, "loss": -0.1213, "num_tokens": 18864044.0, "reward": -0.01562499813735485, "reward_std": 0.2956085801124573, "rewards/rollout_reward_func/mean": -0.01562499813735485, "rewards/rollout_reward_func/std": 0.3173574209213257, "sampling/importance_sampling_ratio/max": 2.1048598289489746, "sampling/importance_sampling_ratio/mean": 0.5130910873413086, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.05569839477539, "sampling/sampling_logp_difference/mean": 2.022254705429077, "step": 421, "step_time": 19.42465589600033 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.002864583395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002864583395421505, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 18.6875, "completions/mean_terminated_length": 17.84000015258789, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1135647781193256, "epoch": 0.00844, "frac_reward_zero_std": 0.0, "grad_norm": 0.527418315410614, "kl": 0.26093520084396005, "learning_rate": 6.880306171506535e-06, "loss": -0.2902, "num_tokens": 18908804.0, "reward": -0.07187500596046448, "reward_std": 0.3746841549873352, "rewards/rollout_reward_func/mean": -0.07187500596046448, "rewards/rollout_reward_func/std": 0.3594210147857666, "sampling/importance_sampling_ratio/max": 2.301598072052002, "sampling/importance_sampling_ratio/mean": 0.5418254137039185, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.29607391357422, "sampling/sampling_logp_difference/mean": 2.277838945388794, "step": 422, "step_time": 19.1148724029772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0036755952751263976, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036755952751263976, "completions/clipped_ratio": 0.25, "completions/max_length": 50.0, "completions/max_terminated_length": 50.0, "completions/mean_length": 21.40625, "completions/mean_terminated_length": 21.20833396911621, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0935792792588472, "epoch": 0.00846, "frac_reward_zero_std": 0.0, "grad_norm": 0.36491483449935913, "kl": 0.37768576736561954, "learning_rate": 6.866876187906582e-06, "loss": -0.2458, "num_tokens": 18952347.0, "reward": 0.02499999850988388, "reward_std": 0.3072707951068878, "rewards/rollout_reward_func/mean": 0.02499999850988388, "rewards/rollout_reward_func/std": 0.3121207356452942, "sampling/importance_sampling_ratio/max": 1.9056066274642944, "sampling/importance_sampling_ratio/mean": 0.4425067901611328, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.680694580078125, "sampling/sampling_logp_difference/mean": 2.0801126956939697, "step": 423, "step_time": 20.43882891003159 }, { "clip_ratio/high_max": 0.020528846187517047, "clip_ratio/high_mean": 0.010264423093758523, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010264423093758523, "completions/clipped_ratio": 0.28125, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 18.3125, "completions/mean_terminated_length": 16.434782028198242, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9159568436443806, "epoch": 0.00848, "frac_reward_zero_std": 0.0, "grad_norm": 0.4247641861438751, "kl": 0.302085661329329, "learning_rate": 6.8534380672927e-06, "loss": -0.064, "num_tokens": 18996417.0, "reward": 0.03437500447034836, "reward_std": 0.3227391242980957, "rewards/rollout_reward_func/mean": 0.03437500447034836, "rewards/rollout_reward_func/std": 0.32489141821861267, "sampling/importance_sampling_ratio/max": 1.5387541055679321, "sampling/importance_sampling_ratio/mean": 0.48763033747673035, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.64442443847656, "sampling/sampling_logp_difference/mean": 1.9538614749908447, "step": 424, "step_time": 19.4860783819895 }, { "clip_ratio/high_max": 0.004185267956927419, "clip_ratio/high_mean": 0.0020926339784637094, "clip_ratio/low_mean": 0.002273478894494474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004366112872958183, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.46875, "completions/mean_terminated_length": 18.920000076293945, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.0256642326712608, "epoch": 0.0085, "frac_reward_zero_std": 0.0, "grad_norm": 0.5202417373657227, "kl": 0.18658088450320065, "learning_rate": 6.839991986922785e-06, "loss": -0.0821, "num_tokens": 19040487.0, "reward": 0.05000000819563866, "reward_std": 0.2815757989883423, "rewards/rollout_reward_func/mean": 0.05000000819563866, "rewards/rollout_reward_func/std": 0.3110854923725128, "sampling/importance_sampling_ratio/max": 2.3305468559265137, "sampling/importance_sampling_ratio/mean": 0.44815394282341003, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.60254669189453, "sampling/sampling_logp_difference/mean": 1.850797414779663, "step": 425, "step_time": 19.021094986004755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00838042062241584, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00838042062241584, "completions/clipped_ratio": 0.1875, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 20.09375, "completions/mean_terminated_length": 19.961538314819336, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1266408059746027, "epoch": 0.00852, "frac_reward_zero_std": 0.0, "grad_norm": 0.4329032003879547, "kl": 0.34652714780531824, "learning_rate": 6.826538124159727e-06, "loss": -0.2211, "num_tokens": 19085360.0, "reward": -0.07500000298023224, "reward_std": 0.3352622091770172, "rewards/rollout_reward_func/mean": -0.07500000298023224, "rewards/rollout_reward_func/std": 0.36010751128196716, "sampling/importance_sampling_ratio/max": 2.523299217224121, "sampling/importance_sampling_ratio/mean": 0.3773546814918518, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.63569259643555, "sampling/sampling_logp_difference/mean": 2.570582866668701, "step": 426, "step_time": 19.574718384988955 }, { "clip_ratio/high_max": 0.005372807150706649, "clip_ratio/high_mean": 0.0026864035753533244, "clip_ratio/low_mean": 0.0024074073880910873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005093810963444412, "completions/clipped_ratio": 0.25, "completions/max_length": 35.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 19.25, "completions/mean_terminated_length": 17.916667938232422, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1556676402688026, "epoch": 0.00854, "frac_reward_zero_std": 0.0, "grad_norm": 0.21690981090068817, "kl": 0.19359170668758452, "learning_rate": 6.813076656469068e-06, "loss": -0.0805, "num_tokens": 19130360.0, "reward": -0.01875000074505806, "reward_std": 0.30824989080429077, "rewards/rollout_reward_func/mean": -0.01875000074505806, "rewards/rollout_reward_func/std": 0.32472074031829834, "sampling/importance_sampling_ratio/max": 2.0114903450012207, "sampling/importance_sampling_ratio/mean": 0.387858510017395, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.688987731933594, "sampling/sampling_logp_difference/mean": 2.2719922065734863, "step": 427, "step_time": 18.95830179900804 }, { "clip_ratio/high_max": 0.008580479072406888, "clip_ratio/high_mean": 0.004290239536203444, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004290239536203444, "completions/clipped_ratio": 0.1875, "completions/max_length": 37.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.53125, "completions/mean_terminated_length": 17.26923179626465, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.937766594812274, "epoch": 0.00856, "frac_reward_zero_std": 0.0, "grad_norm": 0.43421560525894165, "kl": 0.36733805504627526, "learning_rate": 6.799607761416671e-06, "loss": -0.1891, "num_tokens": 19174514.0, "reward": 0.01875000074505806, "reward_std": 0.32610759139060974, "rewards/rollout_reward_func/mean": 0.01875000074505806, "rewards/rollout_reward_func/std": 0.31769075989723206, "sampling/importance_sampling_ratio/max": 2.63346004486084, "sampling/importance_sampling_ratio/mean": 0.4966232180595398, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.430694580078125, "sampling/sampling_logp_difference/mean": 1.9794272184371948, "step": 428, "step_time": 19.122650332996272 }, { "clip_ratio/high_max": 0.016101973596960306, "clip_ratio/high_mean": 0.008050986798480153, "clip_ratio/low_mean": 0.003199404920451343, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01125039195176214, "completions/clipped_ratio": 0.34375, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 20.09375, "completions/mean_terminated_length": 19.809524536132812, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.118963148444891, "epoch": 0.00858, "frac_reward_zero_std": 0.0, "grad_norm": 0.40148282051086426, "kl": 0.2880545621737838, "learning_rate": 6.786131616666364e-06, "loss": -0.0616, "num_tokens": 19219668.0, "reward": 0.09687500447034836, "reward_std": 0.3325503170490265, "rewards/rollout_reward_func/mean": 0.09687500447034836, "rewards/rollout_reward_func/std": 0.34499591588974, "sampling/importance_sampling_ratio/max": 2.227611541748047, "sampling/importance_sampling_ratio/mean": 0.45997732877731323, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.34761428833008, "sampling/sampling_logp_difference/mean": 2.2425947189331055, "step": 429, "step_time": 19.115097595000407 }, { "clip_ratio/high_max": 0.0050322061870247126, "clip_ratio/high_mean": 0.0025161030935123563, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003673510509543121, "completions/clipped_ratio": 0.21875, "completions/max_length": 40.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.9375, "completions/mean_terminated_length": 19.439998626708984, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0482045449316502, "epoch": 0.0086, "frac_reward_zero_std": 0.0, "grad_norm": 0.28823694586753845, "kl": 0.2832900849170983, "learning_rate": 6.772648399977606e-06, "loss": -0.1527, "num_tokens": 19264669.0, "reward": 0.0, "reward_std": 0.3306923508644104, "rewards/rollout_reward_func/mean": 0.0, "rewards/rollout_reward_func/std": 0.3350493609905243, "sampling/importance_sampling_ratio/max": 1.9417836666107178, "sampling/importance_sampling_ratio/mean": 0.37139061093330383, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.345237731933594, "sampling/sampling_logp_difference/mean": 2.24835467338562, "step": 430, "step_time": 19.155119288014248 }, { "clip_ratio/high_max": 0.006623641354963183, "clip_ratio/high_mean": 0.0033118206774815917, "clip_ratio/low_mean": 0.006683778367005289, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009995598928071558, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 21.34375, "completions/mean_terminated_length": 20.153846740722656, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.2675705663859844, "epoch": 0.00862, "frac_reward_zero_std": 0.0, "grad_norm": 0.23281539976596832, "kl": 0.1267843262758106, "learning_rate": 6.759158289203141e-06, "loss": -0.1445, "num_tokens": 19310213.0, "reward": -0.03124999813735485, "reward_std": 0.37799084186553955, "rewards/rollout_reward_func/mean": -0.03124999813735485, "rewards/rollout_reward_func/std": 0.371081680059433, "sampling/importance_sampling_ratio/max": 1.2806048393249512, "sampling/importance_sampling_ratio/mean": 0.24516943097114563, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.523765563964844, "sampling/sampling_logp_difference/mean": 2.891139507293701, "step": 431, "step_time": 19.38663372497831 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.003590129315853119, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005010583903640509, "completions/clipped_ratio": 0.3125, "completions/max_length": 44.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 21.15625, "completions/mean_terminated_length": 19.5, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0175731889903545, "epoch": 0.00864, "frac_reward_zero_std": 0.0, "grad_norm": 0.16050946712493896, "kl": 0.19966782676056027, "learning_rate": 6.745661462286648e-06, "loss": -0.1533, "num_tokens": 19354768.0, "reward": 0.0625000074505806, "reward_std": 0.3236108720302582, "rewards/rollout_reward_func/mean": 0.0625000074505806, "rewards/rollout_reward_func/std": 0.35808631777763367, "sampling/importance_sampling_ratio/max": 1.8984907865524292, "sampling/importance_sampling_ratio/mean": 0.3839946389198303, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.638893127441406, "sampling/sampling_logp_difference/mean": 2.0747387409210205, "step": 432, "step_time": 20.134968788028345 }, { "clip_ratio/high_max": 0.013106740545481443, "clip_ratio/high_mean": 0.006553370272740722, "clip_ratio/low_mean": 0.002474767155945301, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009028137428686023, "completions/clipped_ratio": 0.21875, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 21.15625, "completions/mean_terminated_length": 19.15999984741211, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.2940395027399063, "epoch": 0.00866, "frac_reward_zero_std": 0.0, "grad_norm": 0.31496259570121765, "kl": 0.2525257132947445, "learning_rate": 6.7321580972603996e-06, "loss": -0.0258, "num_tokens": 19401270.0, "reward": -0.08125000447034836, "reward_std": 0.313162237405777, "rewards/rollout_reward_func/mean": -0.08125000447034836, "rewards/rollout_reward_func/std": 0.31769075989723206, "sampling/importance_sampling_ratio/max": 2.941401720046997, "sampling/importance_sampling_ratio/mean": 0.3280535936355591, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.10127639770508, "sampling/sampling_logp_difference/mean": 2.649635076522827, "step": 433, "step_time": 19.88391928600322 }, { "clip_ratio/high_max": 0.010614948812872171, "clip_ratio/high_mean": 0.005307474406436086, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005307474406436086, "completions/clipped_ratio": 0.21875, "completions/max_length": 48.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 23.75, "completions/mean_terminated_length": 22.51999855041504, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1272619981318712, "epoch": 0.00868, "frac_reward_zero_std": 0.0, "grad_norm": 0.27077555656433105, "kl": 0.20322814513929188, "learning_rate": 6.718648372242909e-06, "loss": -0.1219, "num_tokens": 19445957.0, "reward": 3.725290298461914e-09, "reward_std": 0.36182138323783875, "rewards/rollout_reward_func/mean": 3.725290298461914e-09, "rewards/rollout_reward_func/std": 0.3645589351654053, "sampling/importance_sampling_ratio/max": 2.532726526260376, "sampling/importance_sampling_ratio/mean": 0.33740177750587463, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.953617095947266, "sampling/sampling_logp_difference/mean": 2.549910545349121, "step": 434, "step_time": 20.065678059050697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0035063244868069887, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035063244868069887, "completions/clipped_ratio": 0.40625, "completions/max_length": 48.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 21.59375, "completions/mean_terminated_length": 20.526315689086914, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1541830655187368, "epoch": 0.0087, "frac_reward_zero_std": 0.0, "grad_norm": 0.4437900483608246, "kl": 0.6204573467839509, "learning_rate": 6.705132465436579e-06, "loss": -0.0839, "num_tokens": 19490509.0, "reward": -0.006249995902180672, "reward_std": 0.3785223364830017, "rewards/rollout_reward_func/mean": -0.006249995902180672, "rewards/rollout_reward_func/std": 0.3645036220550537, "sampling/importance_sampling_ratio/max": 1.1620644330978394, "sampling/importance_sampling_ratio/mean": 0.2968694567680359, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.50798797607422, "sampling/sampling_logp_difference/mean": 2.1045727729797363, "step": 435, "step_time": 20.08601427600661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 32.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 18.53125, "completions/mean_terminated_length": 17.434782028198242, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8200624957680702, "epoch": 0.00872, "frac_reward_zero_std": 0.0, "grad_norm": 0.3123281002044678, "kl": 0.3322741782758385, "learning_rate": 6.691610555125361e-06, "loss": -0.073, "num_tokens": 19535297.0, "reward": -0.02499999850988388, "reward_std": 0.28244608640670776, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.2794002890586853, "sampling/importance_sampling_ratio/max": 2.010192394256592, "sampling/importance_sampling_ratio/mean": 0.38287898898124695, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.98265075683594, "sampling/sampling_logp_difference/mean": 1.5404489040374756, "step": 436, "step_time": 19.31559296102205 }, { "clip_ratio/high_max": 0.01192985544912517, "clip_ratio/high_mean": 0.005964927724562585, "clip_ratio/low_mean": 0.004014423117041588, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009979350841604173, "completions/clipped_ratio": 0.125, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 20.9375, "completions/mean_terminated_length": 20.5, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0999484229832888, "epoch": 0.00874, "frac_reward_zero_std": 0.0, "grad_norm": 0.3080121576786041, "kl": 0.2732436179649085, "learning_rate": 6.678082819672389e-06, "loss": -0.1043, "num_tokens": 19580195.0, "reward": 0.00937500037252903, "reward_std": 0.2556401193141937, "rewards/rollout_reward_func/mean": 0.00937500037252903, "rewards/rollout_reward_func/std": 0.25445622205734253, "sampling/importance_sampling_ratio/max": 2.2700929641723633, "sampling/importance_sampling_ratio/mean": 0.42031943798065186, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.485870361328125, "sampling/sampling_logp_difference/mean": 2.419626235961914, "step": 437, "step_time": 20.247509792039637 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.0026041667442768812, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003806089865975082, "completions/clipped_ratio": 0.15625, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 21.0625, "completions/mean_terminated_length": 18.925926208496094, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0833143908530474, "epoch": 0.00876, "frac_reward_zero_std": 0.0, "grad_norm": 0.2758292555809021, "kl": 0.29544483916833997, "learning_rate": 6.664549437517642e-06, "loss": -0.1176, "num_tokens": 19625864.0, "reward": -0.0031249970197677612, "reward_std": 0.31409722566604614, "rewards/rollout_reward_func/mean": -0.0031249970197677612, "rewards/rollout_reward_func/std": 0.311587393283844, "sampling/importance_sampling_ratio/max": 1.9121440649032593, "sampling/importance_sampling_ratio/mean": 0.360543429851532, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.836917877197266, "sampling/sampling_logp_difference/mean": 2.023099422454834, "step": 438, "step_time": 20.30503321994911 }, { "clip_ratio/high_max": 0.01248174044303596, "clip_ratio/high_mean": 0.007248934591189027, "clip_ratio/low_mean": 0.0023593305377289653, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009608265128917992, "completions/clipped_ratio": 0.1875, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 22.28125, "completions/mean_terminated_length": 22.538461685180664, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.1385244186967611, "epoch": 0.00878, "frac_reward_zero_std": 0.0, "grad_norm": 0.38884010910987854, "kl": 0.1409922419115901, "learning_rate": 6.651010587175577e-06, "loss": -0.0535, "num_tokens": 19672101.0, "reward": -0.12187500298023224, "reward_std": 0.30086514353752136, "rewards/rollout_reward_func/mean": -0.12187500298023224, "rewards/rollout_reward_func/std": 0.3108099400997162, "sampling/importance_sampling_ratio/max": 1.7346681356430054, "sampling/importance_sampling_ratio/mean": 0.34923315048217773, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.22944641113281, "sampling/sampling_logp_difference/mean": 2.5577492713928223, "step": 439, "step_time": 20.39305087798857 }, { "clip_ratio/high_max": 0.005078125046566129, "clip_ratio/high_mean": 0.0025390625232830644, "clip_ratio/low_mean": 0.002565681468695402, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0051047439919784665, "completions/clipped_ratio": 0.28125, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 21.90625, "completions/mean_terminated_length": 20.34782600402832, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.2778384536504745, "epoch": 0.0088, "frac_reward_zero_std": 0.0, "grad_norm": 0.25430017709732056, "kl": 0.18723285803571343, "learning_rate": 6.637466447232784e-06, "loss": -0.1291, "num_tokens": 19717215.0, "reward": -0.02812499739229679, "reward_std": 0.2967122197151184, "rewards/rollout_reward_func/mean": -0.02812499739229679, "rewards/rollout_reward_func/std": 0.31748443841934204, "sampling/importance_sampling_ratio/max": 2.2730634212493896, "sampling/importance_sampling_ratio/mean": 0.36692124605178833, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.800743103027344, "sampling/sampling_logp_difference/mean": 2.610802173614502, "step": 440, "step_time": 19.50271908902505 }, { "clip_ratio/high_max": 0.0022321429569274187, "clip_ratio/high_mean": 0.0011160714784637094, "clip_ratio/low_mean": 0.0061965902568772435, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007312661735340953, "completions/clipped_ratio": 0.21875, "completions/max_length": 53.0, "completions/max_terminated_length": 53.0, "completions/mean_length": 23.21875, "completions/mean_terminated_length": 22.399999618530273, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0763401798903942, "epoch": 0.00882, "frac_reward_zero_std": 0.0, "grad_norm": 0.22582726180553436, "kl": 0.19894144660793245, "learning_rate": 6.623917196345622e-06, "loss": -0.2443, "num_tokens": 19762284.0, "reward": -0.04062499478459358, "reward_std": 0.34894731640815735, "rewards/rollout_reward_func/mean": -0.04062499478459358, "rewards/rollout_reward_func/std": 0.33969566226005554, "sampling/importance_sampling_ratio/max": 2.279308319091797, "sampling/importance_sampling_ratio/mean": 0.4296965003013611, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.680702209472656, "sampling/sampling_logp_difference/mean": 2.0701406002044678, "step": 441, "step_time": 20.331968670972856 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.0025040064938366413, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003705929615534842, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 18.90625, "completions/mean_terminated_length": 18.481481552124023, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9888889659196138, "epoch": 0.00884, "frac_reward_zero_std": 0.0, "grad_norm": 0.2779117822647095, "kl": 0.21414311043918133, "learning_rate": 6.610363013237871e-06, "loss": -0.1143, "num_tokens": 19806901.0, "reward": 0.012500002980232239, "reward_std": 0.3109426498413086, "rewards/rollout_reward_func/mean": 0.012500002980232239, "rewards/rollout_reward_func/std": 0.3319055438041687, "sampling/importance_sampling_ratio/max": 2.3931376934051514, "sampling/importance_sampling_ratio/mean": 0.5093595385551453, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.89207077026367, "sampling/sampling_logp_difference/mean": 2.073289394378662, "step": 442, "step_time": 18.896356966986787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0032987872255034745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032987872255034745, "completions/clipped_ratio": 0.28125, "completions/max_length": 47.0, "completions/max_terminated_length": 47.0, "completions/mean_length": 22.375, "completions/mean_terminated_length": 20.39130401611328, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0646811909973621, "epoch": 0.00886, "frac_reward_zero_std": 0.0, "grad_norm": 0.25456172227859497, "kl": 0.20140711148269475, "learning_rate": 6.59680407669837e-06, "loss": -0.2385, "num_tokens": 19851221.0, "reward": 0.021875005215406418, "reward_std": 0.3273196518421173, "rewards/rollout_reward_func/mean": 0.021875005215406418, "rewards/rollout_reward_func/std": 0.3260065019130707, "sampling/importance_sampling_ratio/max": 2.857221841812134, "sampling/importance_sampling_ratio/mean": 0.49710601568222046, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.286190032958984, "sampling/sampling_logp_difference/mean": 2.240818977355957, "step": 443, "step_time": 19.952104018026148 }, { "clip_ratio/high_max": 0.010901537723839283, "clip_ratio/high_mean": 0.0054507688619196415, "clip_ratio/low_mean": 0.0009765625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0064273313619196415, "completions/clipped_ratio": 0.09375, "completions/max_length": 61.0, "completions/max_terminated_length": 61.0, "completions/mean_length": 20.90625, "completions/mean_terminated_length": 20.310344696044922, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.8762039951980114, "epoch": 0.00888, "frac_reward_zero_std": 0.0, "grad_norm": 0.3209044933319092, "kl": 0.3158042107243091, "learning_rate": 6.583240565578657e-06, "loss": -0.1549, "num_tokens": 19895897.0, "reward": -0.04374999925494194, "reward_std": 0.33315351605415344, "rewards/rollout_reward_func/mean": -0.04374999925494194, "rewards/rollout_reward_func/std": 0.3330576717853546, "sampling/importance_sampling_ratio/max": 1.4151467084884644, "sampling/importance_sampling_ratio/mean": 0.3686228394508362, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 34.75757598876953, "sampling/sampling_logp_difference/mean": 1.7245910167694092, "step": 444, "step_time": 20.532288414004142 }, { "clip_ratio/high_max": 0.0024999999441206455, "clip_ratio/high_mean": 0.0012499999720603228, "clip_ratio/low_mean": 0.0010416667209938169, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022916666930541396, "completions/clipped_ratio": 0.3125, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 20.5, "completions/mean_terminated_length": 19.22727394104004, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8913105372339487, "epoch": 0.0089, "frac_reward_zero_std": 0.0, "grad_norm": 0.3074992001056671, "kl": 0.21741102705709636, "learning_rate": 6.569672658790611e-06, "loss": -0.2065, "num_tokens": 19941002.0, "reward": -0.05624999850988388, "reward_std": 0.313304603099823, "rewards/rollout_reward_func/mean": -0.05624999850988388, "rewards/rollout_reward_func/std": 0.3242236375808716, "sampling/importance_sampling_ratio/max": 1.9034534692764282, "sampling/importance_sampling_ratio/mean": 0.44770145416259766, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.83198165893555, "sampling/sampling_logp_difference/mean": 1.589620590209961, "step": 445, "step_time": 19.554451431977213 }, { "clip_ratio/high_max": 0.007496081525459886, "clip_ratio/high_mean": 0.003748040762729943, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0059801837196573615, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 21.625, "completions/mean_terminated_length": 20.952381134033203, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.146797452121973, "epoch": 0.00892, "frac_reward_zero_std": 0.0, "grad_norm": 0.32017725706100464, "kl": 0.3756678013596684, "learning_rate": 6.556100535304097e-06, "loss": -0.1264, "num_tokens": 19986482.0, "reward": -0.03125, "reward_std": 0.3420218825340271, "rewards/rollout_reward_func/mean": -0.03125, "rewards/rollout_reward_func/std": 0.3325730562210083, "sampling/importance_sampling_ratio/max": 1.3100520372390747, "sampling/importance_sampling_ratio/mean": 0.2784532606601715, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 35.15132141113281, "sampling/sampling_logp_difference/mean": 2.1222667694091797, "step": 446, "step_time": 18.909556988015538 }, { "clip_ratio/high_max": 0.004166666883975267, "clip_ratio/high_mean": 0.0020833334419876337, "clip_ratio/low_mean": 0.003134300699457526, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005217634257860482, "completions/clipped_ratio": 0.25, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 21.21875, "completions/mean_terminated_length": 19.875, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1436233706772327, "epoch": 0.00894, "frac_reward_zero_std": 0.0, "grad_norm": 0.3320043087005615, "kl": 0.250701422104612, "learning_rate": 6.542524374144598e-06, "loss": -0.2805, "num_tokens": 20030531.0, "reward": -0.04062499478459358, "reward_std": 0.3899208903312683, "rewards/rollout_reward_func/mean": -0.04062499478459358, "rewards/rollout_reward_func/std": 0.37490591406822205, "sampling/importance_sampling_ratio/max": 1.936995506286621, "sampling/importance_sampling_ratio/mean": 0.4298233985900879, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.65372085571289, "sampling/sampling_logp_difference/mean": 2.273745059967041, "step": 447, "step_time": 19.430488469995908 }, { "clip_ratio/high_max": 0.0042261905036866665, "clip_ratio/high_mean": 0.0021130952518433332, "clip_ratio/low_mean": 0.0012019231216982007, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003315018373541534, "completions/clipped_ratio": 0.125, "completions/max_length": 50.0, "completions/max_terminated_length": 50.0, "completions/mean_length": 18.84375, "completions/mean_terminated_length": 18.678571701049805, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9055801760405302, "epoch": 0.00896, "frac_reward_zero_std": 0.0, "grad_norm": 0.43277406692504883, "kl": 0.2575752497650683, "learning_rate": 6.528944354390855e-06, "loss": -0.18, "num_tokens": 20073490.0, "reward": 0.11250001192092896, "reward_std": 0.25741708278656006, "rewards/rollout_reward_func/mean": 0.11250001192092896, "rewards/rollout_reward_func/std": 0.2524077594280243, "sampling/importance_sampling_ratio/max": 2.5589797496795654, "sampling/importance_sampling_ratio/mean": 0.590338945388794, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.382999420166016, "sampling/sampling_logp_difference/mean": 1.9454189538955688, "step": 448, "step_time": 19.538493118001497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0010775862028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010775862028822303, "completions/clipped_ratio": 0.21875, "completions/max_length": 58.0, "completions/max_terminated_length": 58.0, "completions/mean_length": 18.6875, "completions/mean_terminated_length": 18.68000030517578, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0153779685497284, "epoch": 0.00898, "frac_reward_zero_std": 0.0, "grad_norm": 0.3397604823112488, "kl": 0.2849674755707383, "learning_rate": 6.515360655172512e-06, "loss": -0.2592, "num_tokens": 20117581.0, "reward": -0.02499999850988388, "reward_std": 0.359849214553833, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.35650634765625, "sampling/importance_sampling_ratio/max": 2.7575325965881348, "sampling/importance_sampling_ratio/mean": 0.5988928079605103, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.407737731933594, "sampling/sampling_logp_difference/mean": 2.0904178619384766, "step": 449, "step_time": 19.2440325239877 }, { "clip_ratio/high_max": 0.016888233833014965, "clip_ratio/high_mean": 0.008444116916507483, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009802812593989074, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 27.0, "completions/mean_length": 19.1875, "completions/mean_terminated_length": 17.875, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0920599736273289, "epoch": 0.009, "frac_reward_zero_std": 0.0, "grad_norm": 0.2665201723575592, "kl": 0.19036540854722261, "learning_rate": 6.501773455667742e-06, "loss": -0.008, "num_tokens": 20161827.0, "reward": 0.06562499701976776, "reward_std": 0.3260270953178406, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.3158290386199951, "sampling/importance_sampling_ratio/max": 1.3323289155960083, "sampling/importance_sampling_ratio/mean": 0.38135379552841187, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.216339111328125, "sampling/sampling_logp_difference/mean": 2.22908091545105, "step": 450, "step_time": 19.54898253500869 }, { "clip_ratio/high_max": 0.0041082974057644606, "clip_ratio/high_mean": 0.0020541487028822303, "clip_ratio/low_mean": 0.0024519232101738453, "clip_ratio/low_min": 0.0024038462433964014, "clip_ratio/region_mean": 0.004506071913056076, "completions/clipped_ratio": 0.375, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 23.03125, "completions/mean_terminated_length": 22.399999618530273, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1638005264103413, "epoch": 0.00902, "frac_reward_zero_std": 0.0, "grad_norm": 0.31480616331100464, "kl": 0.41079417522996664, "learning_rate": 6.488182935100893e-06, "loss": -0.2049, "num_tokens": 20207920.0, "reward": 0.04374999925494194, "reward_std": 0.2849835753440857, "rewards/rollout_reward_func/mean": 0.04374999925494194, "rewards/rollout_reward_func/std": 0.2861733138561249, "sampling/importance_sampling_ratio/max": 1.691328525543213, "sampling/importance_sampling_ratio/mean": 0.2865769863128662, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.48747634887695, "sampling/sampling_logp_difference/mean": 2.1833622455596924, "step": 451, "step_time": 19.01770348098944 }, { "clip_ratio/high_max": 0.014887659111991525, "clip_ratio/high_mean": 0.007443829555995762, "clip_ratio/low_mean": 0.0025584795512259007, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010002309107221663, "completions/clipped_ratio": 0.09375, "completions/max_length": 63.0, "completions/max_terminated_length": 63.0, "completions/mean_length": 21.28125, "completions/mean_terminated_length": 21.827585220336914, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.030045922845602, "epoch": 0.00904, "frac_reward_zero_std": 0.0, "grad_norm": 0.2342664748430252, "kl": 0.4698551546316594, "learning_rate": 6.474589272740116e-06, "loss": -0.1125, "num_tokens": 20252154.0, "reward": 0.02812500298023224, "reward_std": 0.3442396819591522, "rewards/rollout_reward_func/mean": 0.02812500298023224, "rewards/rollout_reward_func/std": 0.34569647908210754, "sampling/importance_sampling_ratio/max": 1.2068356275558472, "sampling/importance_sampling_ratio/mean": 0.3921709656715393, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.83694839477539, "sampling/sampling_logp_difference/mean": 2.4936487674713135, "step": 452, "step_time": 19.826935456017964 }, { "clip_ratio/high_max": 0.005291005363687873, "clip_ratio/high_mean": 0.0026455026818439364, "clip_ratio/low_mean": 0.002813697326928377, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005459200008772314, "completions/clipped_ratio": 0.15625, "completions/max_length": 73.0, "completions/max_terminated_length": 73.0, "completions/mean_length": 21.53125, "completions/mean_terminated_length": 20.814815521240234, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.066960671916604, "epoch": 0.00906, "frac_reward_zero_std": 0.0, "grad_norm": 0.27967438101768494, "kl": 0.34747069189324975, "learning_rate": 6.46099264789501e-06, "loss": -0.1932, "num_tokens": 20297615.0, "reward": -0.09062500298023224, "reward_std": 0.3312755823135376, "rewards/rollout_reward_func/mean": -0.09062500298023224, "rewards/rollout_reward_func/std": 0.33731329441070557, "sampling/importance_sampling_ratio/max": 1.7108696699142456, "sampling/importance_sampling_ratio/mean": 0.33838966488838196, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.258827209472656, "sampling/sampling_logp_difference/mean": 2.1590476036071777, "step": 453, "step_time": 20.40898378106067 }, { "clip_ratio/high_max": 0.007499999832361937, "clip_ratio/high_mean": 0.0047580646350979805, "clip_ratio/low_mean": 0.0011160714784637094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00587413611356169, "completions/clipped_ratio": 0.15625, "completions/max_length": 47.0, "completions/max_terminated_length": 47.0, "completions/mean_length": 22.15625, "completions/mean_terminated_length": 22.851852416992188, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0601956564933062, "epoch": 0.00908, "frac_reward_zero_std": 0.0, "grad_norm": 0.1353680044412613, "kl": 0.2347452596295625, "learning_rate": 6.447393239914247e-06, "loss": -0.1301, "num_tokens": 20341760.0, "reward": 0.19375000894069672, "reward_std": 0.2538323998451233, "rewards/rollout_reward_func/mean": 0.19375000894069672, "rewards/rollout_reward_func/std": 0.25770387053489685, "sampling/importance_sampling_ratio/max": 1.853577971458435, "sampling/importance_sampling_ratio/mean": 0.4109792113304138, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.87886428833008, "sampling/sampling_logp_difference/mean": 2.256756544113159, "step": 454, "step_time": 20.131608704017708 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0008928571478463709, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020502645638771355, "completions/clipped_ratio": 0.15625, "completions/max_length": 35.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 21.5, "completions/mean_terminated_length": 20.851852416992188, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1210005059838295, "epoch": 0.0091, "frac_reward_zero_std": 0.0, "grad_norm": 0.3534860610961914, "kl": 0.12591948849149048, "learning_rate": 6.433791228183211e-06, "loss": -0.0501, "num_tokens": 20387281.0, "reward": 0.018750004470348358, "reward_std": 0.3033848702907562, "rewards/rollout_reward_func/mean": 0.018750004470348358, "rewards/rollout_reward_func/std": 0.33643049001693726, "sampling/importance_sampling_ratio/max": 1.1795791387557983, "sampling/importance_sampling_ratio/mean": 0.31849831342697144, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.45127868652344, "sampling/sampling_logp_difference/mean": 2.2073798179626465, "step": 455, "step_time": 19.673035395957413 }, { "clip_ratio/high_max": 0.007525083841755986, "clip_ratio/high_mean": 0.003762541920877993, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003762541920877993, "completions/clipped_ratio": 0.3125, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 19.84375, "completions/mean_terminated_length": 19.40909194946289, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8527903873473406, "epoch": 0.00912, "frac_reward_zero_std": 0.0, "grad_norm": 0.3173692524433136, "kl": 0.5284320106729865, "learning_rate": 6.42018679212163e-06, "loss": -0.1423, "num_tokens": 20431365.0, "reward": -0.03437500447034836, "reward_std": 0.2875387668609619, "rewards/rollout_reward_func/mean": -0.03437500447034836, "rewards/rollout_reward_func/std": 0.31377965211868286, "sampling/importance_sampling_ratio/max": 1.5395240783691406, "sampling/importance_sampling_ratio/mean": 0.38534605503082275, "sampling/importance_sampling_ratio/min": 2.1019476964872256e-44, "sampling/sampling_logp_difference/max": 33.410118103027344, "sampling/sampling_logp_difference/mean": 1.462509274482727, "step": 456, "step_time": 19.787334724969696 }, { "clip_ratio/high_max": 0.010146104265004396, "clip_ratio/high_mean": 0.005073052132502198, "clip_ratio/low_mean": 0.0022916666930541396, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007364718825556338, "completions/clipped_ratio": 0.375, "completions/max_length": 64.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 22.34375, "completions/mean_terminated_length": 17.899999618530273, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9189662598073483, "epoch": 0.00914, "frac_reward_zero_std": 0.0, "grad_norm": 0.20450185239315033, "kl": 0.18942619278095663, "learning_rate": 6.406580111181216e-06, "loss": -0.1937, "num_tokens": 20475787.0, "reward": -0.06562499701976776, "reward_std": 0.4188818633556366, "rewards/rollout_reward_func/mean": -0.06562499701976776, "rewards/rollout_reward_func/std": 0.41551318764686584, "sampling/importance_sampling_ratio/max": 1.5192030668258667, "sampling/importance_sampling_ratio/mean": 0.3587196469306946, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.298370361328125, "sampling/sampling_logp_difference/mean": 1.6076300144195557, "step": 457, "step_time": 20.10030905304302 }, { "clip_ratio/high_max": 0.005121237598359585, "clip_ratio/high_mean": 0.0025606187991797924, "clip_ratio/low_mean": 0.0030795882339589298, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005640207033138722, "completions/clipped_ratio": 0.09375, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 22.875, "completions/mean_terminated_length": 22.758621215820312, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1110735163092613, "epoch": 0.00916, "frac_reward_zero_std": 0.0, "grad_norm": 0.24189449846744537, "kl": 0.1198890961240977, "learning_rate": 6.392971364843287e-06, "loss": -0.1822, "num_tokens": 20521597.0, "reward": -0.15312498807907104, "reward_std": 0.3741160035133362, "rewards/rollout_reward_func/mean": -0.15312498807907104, "rewards/rollout_reward_func/std": 0.3645451068878174, "sampling/importance_sampling_ratio/max": 1.8913145065307617, "sampling/importance_sampling_ratio/mean": 0.3600963354110718, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.727577209472656, "sampling/sampling_logp_difference/mean": 2.4634037017822266, "step": 458, "step_time": 21.15259795897873 }, { "clip_ratio/high_max": 0.008453207788988948, "clip_ratio/high_mean": 0.004226603894494474, "clip_ratio/low_mean": 0.0058407888282090425, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010067392722703516, "completions/clipped_ratio": 0.21875, "completions/max_length": 60.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 21.96875, "completions/mean_terminated_length": 20.239999771118164, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.038379244506359, "epoch": 0.00918, "frac_reward_zero_std": 0.0, "grad_norm": 0.3448192775249481, "kl": 0.34345165779814124, "learning_rate": 6.379360732616405e-06, "loss": -0.2711, "num_tokens": 20565715.0, "reward": -0.05624999850988388, "reward_std": 0.3823188245296478, "rewards/rollout_reward_func/mean": -0.05624999850988388, "rewards/rollout_reward_func/std": 0.3934442400932312, "sampling/importance_sampling_ratio/max": 1.892703890800476, "sampling/importance_sampling_ratio/mean": 0.5118710398674011, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.435089111328125, "sampling/sampling_logp_difference/mean": 2.16616153717041, "step": 459, "step_time": 19.94590993800375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.625, "completions/mean_terminated_length": 19.11111068725586, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9184741135686636, "epoch": 0.0092, "frac_reward_zero_std": 0.0, "grad_norm": 0.4425618648529053, "kl": 0.2301771577913314, "learning_rate": 6.365748394034013e-06, "loss": -0.3101, "num_tokens": 20609137.0, "reward": 0.02187500335276127, "reward_std": 0.3589993715286255, "rewards/rollout_reward_func/mean": 0.02187500335276127, "rewards/rollout_reward_func/std": 0.3589720129966736, "sampling/importance_sampling_ratio/max": 2.261110782623291, "sampling/importance_sampling_ratio/mean": 0.6636996865272522, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.09524154663086, "sampling/sampling_logp_difference/mean": 1.7645838260650635, "step": 460, "step_time": 19.6064179000241 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.004882564244326204, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006039971660356969, "completions/clipped_ratio": 0.1875, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 20.34375, "completions/mean_terminated_length": 20.423078536987305, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.061398759484291, "epoch": 0.00922, "frac_reward_zero_std": 0.0, "grad_norm": 0.5765213966369629, "kl": 0.20782124437391758, "learning_rate": 6.352134528652057e-06, "loss": -0.2454, "num_tokens": 20653666.0, "reward": -0.02812500298023224, "reward_std": 0.3492008149623871, "rewards/rollout_reward_func/mean": -0.02812500298023224, "rewards/rollout_reward_func/std": 0.3603174090385437, "sampling/importance_sampling_ratio/max": 2.05844783782959, "sampling/importance_sampling_ratio/mean": 0.4572557508945465, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.039241790771484, "sampling/sampling_logp_difference/mean": 2.2426540851593018, "step": 461, "step_time": 20.581990901991958 }, { "clip_ratio/high_max": 0.007411859114654362, "clip_ratio/high_mean": 0.003705929557327181, "clip_ratio/low_mean": 0.00231799460016191, "clip_ratio/low_min": 0.0022321429569274187, "clip_ratio/region_mean": 0.006023924157489091, "completions/clipped_ratio": 0.1875, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 22.46875, "completions/mean_terminated_length": 22.230770111083984, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0017934869974852, "epoch": 0.00924, "frac_reward_zero_std": 0.0, "grad_norm": 0.2035132348537445, "kl": 0.31587542174384, "learning_rate": 6.3385193160466255e-06, "loss": -0.054, "num_tokens": 20699246.0, "reward": 0.0031250007450580597, "reward_std": 0.31590592861175537, "rewards/rollout_reward_func/mean": 0.0031250007450580597, "rewards/rollout_reward_func/std": 0.314677894115448, "sampling/importance_sampling_ratio/max": 1.3976414203643799, "sampling/importance_sampling_ratio/mean": 0.24446028470993042, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.70059585571289, "sampling/sampling_logp_difference/mean": 1.9756922721862793, "step": 462, "step_time": 20.18505993002327 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0025040064938366413, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003862702171318233, "completions/clipped_ratio": 0.21875, "completions/max_length": 30.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 17.03125, "completions/mean_terminated_length": 17.0, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.8773078303784132, "epoch": 0.00926, "frac_reward_zero_std": 0.0, "grad_norm": 0.28921717405319214, "kl": 0.3202285678125918, "learning_rate": 6.324902935811576e-06, "loss": -0.1885, "num_tokens": 20743354.0, "reward": 0.009375002235174179, "reward_std": 0.2958565950393677, "rewards/rollout_reward_func/mean": 0.009375002235174179, "rewards/rollout_reward_func/std": 0.3104204833507538, "sampling/importance_sampling_ratio/max": 2.0807769298553467, "sampling/importance_sampling_ratio/mean": 0.5052824020385742, "sampling/importance_sampling_ratio/min": 5.885453550164232e-44, "sampling/sampling_logp_difference/max": 33.68898391723633, "sampling/sampling_logp_difference/mean": 1.7365301847457886, "step": 463, "step_time": 18.442722270003287 }, { "clip_ratio/high_max": 0.009189189062453806, "clip_ratio/high_mean": 0.004594594531226903, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004594594531226903, "completions/clipped_ratio": 0.34375, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 22.71875, "completions/mean_terminated_length": 22.666667938232422, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.2084594927728176, "epoch": 0.00928, "frac_reward_zero_std": 0.0, "grad_norm": 0.29821422696113586, "kl": 0.20725162874441594, "learning_rate": 6.311285567556168e-06, "loss": -0.0765, "num_tokens": 20789569.0, "reward": 0.04374999552965164, "reward_std": 0.2663727104663849, "rewards/rollout_reward_func/mean": 0.04374999552965164, "rewards/rollout_reward_func/std": 0.30367374420166016, "sampling/importance_sampling_ratio/max": 1.5720722675323486, "sampling/importance_sampling_ratio/mean": 0.27697378396987915, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.722618103027344, "sampling/sampling_logp_difference/mean": 2.2516682147979736, "step": 464, "step_time": 19.897026617982192 }, { "clip_ratio/high_max": 0.0047186610754579306, "clip_ratio/high_mean": 0.0023593305377289653, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023593305377289653, "completions/clipped_ratio": 0.25, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 22.375, "completions/mean_terminated_length": 20.0, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.0490356255322695, "epoch": 0.0093, "frac_reward_zero_std": 0.0, "grad_norm": 0.3611075282096863, "kl": 0.166987614473328, "learning_rate": 6.297667390902694e-06, "loss": -0.0609, "num_tokens": 20834817.0, "reward": -0.02187499590218067, "reward_std": 0.35289180278778076, "rewards/rollout_reward_func/mean": -0.02187499590218067, "rewards/rollout_reward_func/std": 0.3580722510814667, "sampling/importance_sampling_ratio/max": 2.4606499671936035, "sampling/importance_sampling_ratio/mean": 0.49369049072265625, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.035118103027344, "sampling/sampling_logp_difference/mean": 2.0621371269226074, "step": 465, "step_time": 20.449602665001294 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025161030935123563, "completions/clipped_ratio": 0.34375, "completions/max_length": 48.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 22.71875, "completions/mean_terminated_length": 20.33333396911621, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.0353582110255957, "epoch": 0.00932, "frac_reward_zero_std": 0.0, "grad_norm": 0.2681506872177124, "kl": 0.16360582667402923, "learning_rate": 6.284048585484113e-06, "loss": -0.2623, "num_tokens": 20879285.0, "reward": -0.10624999552965164, "reward_std": 0.3945175111293793, "rewards/rollout_reward_func/mean": -0.10624999552965164, "rewards/rollout_reward_func/std": 0.39097684621810913, "sampling/importance_sampling_ratio/max": 1.8585519790649414, "sampling/importance_sampling_ratio/mean": 0.41516733169555664, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 34.40132522583008, "sampling/sampling_logp_difference/mean": 1.9461851119995117, "step": 466, "step_time": 20.56234819800011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.002582282933872193, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002582282933872193, "completions/clipped_ratio": 0.125, "completions/max_length": 42.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 21.15625, "completions/mean_terminated_length": 19.35714340209961, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.129486870020628, "epoch": 0.00934, "frac_reward_zero_std": 0.0, "grad_norm": 0.22022448480129242, "kl": 0.195892212446779, "learning_rate": 6.270429330941671e-06, "loss": -0.1936, "num_tokens": 20924285.0, "reward": -0.04062499850988388, "reward_std": 0.3821835219860077, "rewards/rollout_reward_func/mean": -0.04062499850988388, "rewards/rollout_reward_func/std": 0.39664673805236816, "sampling/importance_sampling_ratio/max": 2.065772294998169, "sampling/importance_sampling_ratio/mean": 0.3910067677497864, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.00494384765625, "sampling/sampling_logp_difference/mean": 2.5391340255737305, "step": 467, "step_time": 20.149085357043077 }, { "clip_ratio/high_max": 0.010635407641530037, "clip_ratio/high_mean": 0.0053177038207650185, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0053177038207650185, "completions/clipped_ratio": 0.125, "completions/max_length": 48.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 20.65625, "completions/mean_terminated_length": 19.35714340209961, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.2290124595165253, "epoch": 0.00936, "frac_reward_zero_std": 0.0, "grad_norm": 0.21549274027347565, "kl": 0.2294850426260382, "learning_rate": 6.2568098069225436e-06, "loss": -0.1366, "num_tokens": 20970021.0, "reward": -0.03749999403953552, "reward_std": 0.33297187089920044, "rewards/rollout_reward_func/mean": -0.03749999403953552, "rewards/rollout_reward_func/std": 0.3571843206882477, "sampling/importance_sampling_ratio/max": 1.3424831628799438, "sampling/importance_sampling_ratio/mean": 0.31097716093063354, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.80569839477539, "sampling/sampling_logp_difference/mean": 2.635180950164795, "step": 468, "step_time": 20.565222904013353 }, { "clip_ratio/high_max": 0.009131865110248327, "clip_ratio/high_mean": 0.004565932555124164, "clip_ratio/low_mean": 0.0029085498536005616, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007474482408724725, "completions/clipped_ratio": 0.21875, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 19.6875, "completions/mean_terminated_length": 18.479999542236328, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0005505811423063, "epoch": 0.00938, "frac_reward_zero_std": 0.0, "grad_norm": 0.4864816963672638, "kl": 0.31818936998024583, "learning_rate": 6.243190193077457e-06, "loss": -0.1303, "num_tokens": 21014829.0, "reward": -0.00624999962747097, "reward_std": 0.26017123460769653, "rewards/rollout_reward_func/mean": -0.00624999962747097, "rewards/rollout_reward_func/std": 0.2626631557941437, "sampling/importance_sampling_ratio/max": 1.4015250205993652, "sampling/importance_sampling_ratio/mean": 0.3884854018688202, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.848690032958984, "sampling/sampling_logp_difference/mean": 1.839158058166504, "step": 469, "step_time": 19.401759368003695 }, { "clip_ratio/high_max": 0.005104166688397527, "clip_ratio/high_mean": 0.0025520833441987634, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003709490760229528, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.375, "completions/mean_terminated_length": 19.71428680419922, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1440453194081783, "epoch": 0.0094, "frac_reward_zero_std": 0.0, "grad_norm": 0.26467373967170715, "kl": 0.2544976621866226, "learning_rate": 6.2295706690583325e-06, "loss": -0.0707, "num_tokens": 21059874.0, "reward": 0.06562500447034836, "reward_std": 0.23422469198703766, "rewards/rollout_reward_func/mean": 0.06562500447034836, "rewards/rollout_reward_func/std": 0.2924968898296356, "sampling/importance_sampling_ratio/max": 2.636164426803589, "sampling/importance_sampling_ratio/mean": 0.34043288230895996, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.835731506347656, "sampling/sampling_logp_difference/mean": 2.269341468811035, "step": 470, "step_time": 19.765122032011277 }, { "clip_ratio/high_max": 0.007523148320615292, "clip_ratio/high_mean": 0.003761574160307646, "clip_ratio/low_mean": 0.0067830217885784805, "clip_ratio/low_min": 0.0026041667442768812, "clip_ratio/region_mean": 0.010544595948886126, "completions/clipped_ratio": 0.3125, "completions/max_length": 34.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 18.5, "completions/mean_terminated_length": 16.72727394104004, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.8282687440514565, "epoch": 0.00942, "frac_reward_zero_std": 0.0, "grad_norm": 0.2507951855659485, "kl": 0.26296984660439193, "learning_rate": 6.215951414515888e-06, "loss": -0.1136, "num_tokens": 21103390.0, "reward": 0.06562499701976776, "reward_std": 0.3097735643386841, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.339458167552948, "sampling/importance_sampling_ratio/max": 1.6955907344818115, "sampling/importance_sampling_ratio/mean": 0.41872087121009827, "sampling/importance_sampling_ratio/min": 1.6114932339735396e-43, "sampling/sampling_logp_difference/max": 33.09748077392578, "sampling/sampling_logp_difference/mean": 1.4301233291625977, "step": 471, "step_time": 19.1396704929939 }, { "clip_ratio/high_max": 0.0070398354437202215, "clip_ratio/high_mean": 0.0035199177218601108, "clip_ratio/low_mean": 0.0010080644860863686, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0045279820915311575, "completions/clipped_ratio": 0.34375, "completions/max_length": 34.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 20.8125, "completions/mean_terminated_length": 19.238096237182617, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.105453073978424, "epoch": 0.00944, "frac_reward_zero_std": 0.0, "grad_norm": 0.27048707008361816, "kl": 0.20575883565470576, "learning_rate": 6.202332609097308e-06, "loss": -0.1192, "num_tokens": 21148104.0, "reward": 0.04062499850988388, "reward_std": 0.26445505023002625, "rewards/rollout_reward_func/mean": 0.04062499850988388, "rewards/rollout_reward_func/std": 0.27691778540611267, "sampling/importance_sampling_ratio/max": 1.2898048162460327, "sampling/importance_sampling_ratio/mean": 0.27261248230934143, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.74236297607422, "sampling/sampling_logp_difference/mean": 2.146104097366333, "step": 472, "step_time": 19.54405046804459 }, { "clip_ratio/high_max": 0.005372807150706649, "clip_ratio/high_mean": 0.0026864035753533244, "clip_ratio/low_mean": 0.0017407162231393158, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00442711979849264, "completions/clipped_ratio": 0.21875, "completions/max_length": 58.0, "completions/max_terminated_length": 58.0, "completions/mean_length": 21.375, "completions/mean_terminated_length": 19.760000228881836, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0522272773087025, "epoch": 0.00946, "frac_reward_zero_std": 0.0, "grad_norm": 0.2835557460784912, "kl": 0.4556380035355687, "learning_rate": 6.188714432443833e-06, "loss": -0.1753, "num_tokens": 21192520.0, "reward": 0.009375001303851604, "reward_std": 0.3684445023536682, "rewards/rollout_reward_func/mean": 0.009375001303851604, "rewards/rollout_reward_func/std": 0.35592618584632874, "sampling/importance_sampling_ratio/max": 2.1682493686676025, "sampling/importance_sampling_ratio/mean": 0.41947343945503235, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.22261428833008, "sampling/sampling_logp_difference/mean": 2.0161445140838623, "step": 473, "step_time": 20.138958174953586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.002760808332823217, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002760808332823217, "completions/clipped_ratio": 0.25, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 21.125, "completions/mean_terminated_length": 21.375, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.1282879374921322, "epoch": 0.00948, "frac_reward_zero_std": 0.0, "grad_norm": 0.34423431754112244, "kl": 0.20724525721743703, "learning_rate": 6.175097064188427e-06, "loss": -0.1327, "num_tokens": 21238150.0, "reward": -0.09062500298023224, "reward_std": 0.2896912097930908, "rewards/rollout_reward_func/mean": -0.09062500298023224, "rewards/rollout_reward_func/std": 0.2922210693359375, "sampling/importance_sampling_ratio/max": 1.3738524913787842, "sampling/importance_sampling_ratio/mean": 0.3517031669616699, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.420127868652344, "sampling/sampling_logp_difference/mean": 2.1720333099365234, "step": 474, "step_time": 20.04231938497105 }, { "clip_ratio/high_max": 0.00633208267390728, "clip_ratio/high_mean": 0.00316604133695364, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00316604133695364, "completions/clipped_ratio": 0.375, "completions/max_length": 48.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 22.0, "completions/mean_terminated_length": 19.399999618530273, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.893897645175457, "epoch": 0.0095, "frac_reward_zero_std": 0.0, "grad_norm": 0.3680660128593445, "kl": 0.1591394143179059, "learning_rate": 6.161480683953376e-06, "loss": -0.1623, "num_tokens": 21283594.0, "reward": 1.862645149230957e-09, "reward_std": 0.34307628870010376, "rewards/rollout_reward_func/mean": 1.862645149230957e-09, "rewards/rollout_reward_func/std": 0.369829922914505, "sampling/importance_sampling_ratio/max": 2.570244073867798, "sampling/importance_sampling_ratio/mean": 0.578948974609375, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.3295783996582, "sampling/sampling_logp_difference/mean": 1.536694884300232, "step": 475, "step_time": 20.193887811008608 }, { "clip_ratio/high_max": 0.008181818295270205, "clip_ratio/high_mean": 0.004090909147635102, "clip_ratio/low_mean": 0.0019602272659540176, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00605113641358912, "completions/clipped_ratio": 0.125, "completions/max_length": 48.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 20.46875, "completions/mean_terminated_length": 18.892858505249023, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.974602609872818, "epoch": 0.00952, "frac_reward_zero_std": 0.0, "grad_norm": 0.35765036940574646, "kl": 0.25093722995370626, "learning_rate": 6.147865471347945e-06, "loss": -0.082, "num_tokens": 21328433.0, "reward": -0.01249999925494194, "reward_std": 0.30121690034866333, "rewards/rollout_reward_func/mean": -0.01249999925494194, "rewards/rollout_reward_func/std": 0.2959402799606323, "sampling/importance_sampling_ratio/max": 2.2705259323120117, "sampling/importance_sampling_ratio/mean": 0.4569959044456482, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.278831481933594, "sampling/sampling_logp_difference/mean": 1.958507776260376, "step": 476, "step_time": 19.865013542963425 }, { "clip_ratio/high_max": 0.0016891892300918698, "clip_ratio/high_mean": 0.0008445946150459349, "clip_ratio/low_mean": 0.002719907439313829, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003564502054359764, "completions/clipped_ratio": 0.1875, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 20.46875, "completions/mean_terminated_length": 19.76923179626465, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.9531599506735802, "epoch": 0.00954, "frac_reward_zero_std": 0.0, "grad_norm": 0.27161818742752075, "kl": 0.30334981670603156, "learning_rate": 6.134251605965988e-06, "loss": -0.0803, "num_tokens": 21373804.0, "reward": -0.08437500149011612, "reward_std": 0.32007279992103577, "rewards/rollout_reward_func/mean": -0.08437500149011612, "rewards/rollout_reward_func/std": 0.3193838596343994, "sampling/importance_sampling_ratio/max": 1.76918363571167, "sampling/importance_sampling_ratio/mean": 0.3453041911125183, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.348690032958984, "sampling/sampling_logp_difference/mean": 1.9059884548187256, "step": 477, "step_time": 20.17838048702106 }, { "clip_ratio/high_max": 0.010720601421780884, "clip_ratio/high_mean": 0.005360300710890442, "clip_ratio/low_mean": 0.003062213188968599, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008422513899859041, "completions/clipped_ratio": 0.3125, "completions/max_length": 39.0, "completions/max_terminated_length": 39.0, "completions/mean_length": 23.8125, "completions/mean_terminated_length": 21.545455932617188, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.357244135811925, "epoch": 0.00956, "frac_reward_zero_std": 0.0, "grad_norm": 0.18058575689792633, "kl": 0.2077388558536768, "learning_rate": 6.1206392673835955e-06, "loss": -0.0618, "num_tokens": 21420557.0, "reward": -0.109375, "reward_std": 0.3795943558216095, "rewards/rollout_reward_func/mean": -0.109375, "rewards/rollout_reward_func/std": 0.37014326453208923, "sampling/importance_sampling_ratio/max": 1.447843313217163, "sampling/importance_sampling_ratio/mean": 0.1693965345621109, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.58694839477539, "sampling/sampling_logp_difference/mean": 2.804283857345581, "step": 478, "step_time": 20.98788051099109 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.34375, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 22.78125, "completions/mean_terminated_length": 21.238096237182617, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1051406525075436, "epoch": 0.00958, "frac_reward_zero_std": 0.0, "grad_norm": 0.3806212842464447, "kl": 0.2538395645096898, "learning_rate": 6.1070286351567154e-06, "loss": -0.0902, "num_tokens": 21466191.0, "reward": -0.02499999664723873, "reward_std": 0.37310171127319336, "rewards/rollout_reward_func/mean": -0.02499999664723873, "rewards/rollout_reward_func/std": 0.3810003697872162, "sampling/importance_sampling_ratio/max": 2.016733169555664, "sampling/importance_sampling_ratio/mean": 0.35455065965652466, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 34.039249420166016, "sampling/sampling_logp_difference/mean": 1.8610315322875977, "step": 479, "step_time": 20.224054947975674 }, { "clip_ratio/high_max": 0.0050322061870247126, "clip_ratio/high_mean": 0.0025161030935123563, "clip_ratio/low_mean": 0.003923382726497948, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0064394858200103045, "completions/clipped_ratio": 0.21875, "completions/max_length": 34.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 19.03125, "completions/mean_terminated_length": 17.639999389648438, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0010151267051697, "epoch": 0.0096, "frac_reward_zero_std": 0.0, "grad_norm": 0.31834182143211365, "kl": 0.24302169517613947, "learning_rate": 6.093419888818785e-06, "loss": -0.1994, "num_tokens": 21510253.0, "reward": 5.587935447692871e-09, "reward_std": 0.34868937730789185, "rewards/rollout_reward_func/mean": 5.587935447692871e-09, "rewards/rollout_reward_func/std": 0.34172236919403076, "sampling/importance_sampling_ratio/max": 1.8464226722717285, "sampling/importance_sampling_ratio/mean": 0.4422557055950165, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.473690032958984, "sampling/sampling_logp_difference/mean": 2.105111598968506, "step": 480, "step_time": 19.617590524998377 }, { "clip_ratio/high_max": 0.007535209180787206, "clip_ratio/high_mean": 0.003767604590393603, "clip_ratio/low_mean": 0.004487179685384035, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008254784275777638, "completions/clipped_ratio": 0.1875, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 19.84375, "completions/mean_terminated_length": 19.19230842590332, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.0252085458487272, "epoch": 0.00962, "frac_reward_zero_std": 0.0, "grad_norm": 0.465994656085968, "kl": 0.138520288746804, "learning_rate": 6.0798132078783714e-06, "loss": -0.0486, "num_tokens": 21554369.0, "reward": 0.09375, "reward_std": 0.23668953776359558, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.23409055173397064, "sampling/importance_sampling_ratio/max": 1.563858151435852, "sampling/importance_sampling_ratio/mean": 0.45592641830444336, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.45138931274414, "sampling/sampling_logp_difference/mean": 2.1921470165252686, "step": 481, "step_time": 19.43110142200021 }, { "clip_ratio/high_max": 0.009123641299083829, "clip_ratio/high_mean": 0.0045618206495419145, "clip_ratio/low_mean": 0.0030307112028822303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007592531852424145, "completions/clipped_ratio": 0.1875, "completions/max_length": 31.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 17.875, "completions/mean_terminated_length": 17.615385055541992, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.0876262113451958, "epoch": 0.00964, "frac_reward_zero_std": 0.0, "grad_norm": 0.3745656907558441, "kl": 0.17907293676398695, "learning_rate": 6.0662087718167915e-06, "loss": -0.1182, "num_tokens": 21598413.0, "reward": -0.01562500186264515, "reward_std": 0.3526502251625061, "rewards/rollout_reward_func/mean": -0.01562500186264515, "rewards/rollout_reward_func/std": 0.354791522026062, "sampling/importance_sampling_ratio/max": 2.2343592643737793, "sampling/importance_sampling_ratio/mean": 0.5006654858589172, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 34.547367095947266, "sampling/sampling_logp_difference/mean": 2.3383991718292236, "step": 482, "step_time": 19.123759515976417 }, { "clip_ratio/high_max": 0.007996633183211088, "clip_ratio/high_mean": 0.003998316591605544, "clip_ratio/low_mean": 0.004673191579058766, "clip_ratio/low_min": 0.0015625000232830644, "clip_ratio/region_mean": 0.00867150817066431, "completions/clipped_ratio": 0.1875, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 19.8125, "completions/mean_terminated_length": 17.653846740722656, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0858865529298782, "epoch": 0.00966, "frac_reward_zero_std": 0.0, "grad_norm": 0.34182730317115784, "kl": 0.15315577411092818, "learning_rate": 6.0526067600857556e-06, "loss": -0.0335, "num_tokens": 21643723.0, "reward": -0.06874999403953552, "reward_std": 0.3828848898410797, "rewards/rollout_reward_func/mean": -0.06874999403953552, "rewards/rollout_reward_func/std": 0.39056411385536194, "sampling/importance_sampling_ratio/max": 1.699094533920288, "sampling/importance_sampling_ratio/mean": 0.40946057438850403, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.67106628417969, "sampling/sampling_logp_difference/mean": 2.3931195735931396, "step": 483, "step_time": 19.482643166033085 }, { "clip_ratio/high_max": 0.00837623723782599, "clip_ratio/high_mean": 0.004188118618912995, "clip_ratio/low_mean": 0.002464657765813172, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006652776384726167, "completions/clipped_ratio": 0.21875, "completions/max_length": 51.0, "completions/max_terminated_length": 51.0, "completions/mean_length": 20.4375, "completions/mean_terminated_length": 19.31999969482422, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9835079293698072, "epoch": 0.00968, "frac_reward_zero_std": 0.0, "grad_norm": 0.4093114733695984, "kl": 0.33341193688102067, "learning_rate": 6.039007352104992e-06, "loss": -0.2074, "num_tokens": 21688431.0, "reward": 0.08124999701976776, "reward_std": 0.29798752069473267, "rewards/rollout_reward_func/mean": 0.08124999701976776, "rewards/rollout_reward_func/std": 0.3187045454978943, "sampling/importance_sampling_ratio/max": 2.6371381282806396, "sampling/importance_sampling_ratio/mean": 0.5411588549613953, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.53273391723633, "sampling/sampling_logp_difference/mean": 1.959312915802002, "step": 484, "step_time": 20.381813172018155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0035370769910514355, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035370769910514355, "completions/clipped_ratio": 0.21875, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 21.21875, "completions/mean_terminated_length": 19.0, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0533181577920914, "epoch": 0.0097, "frac_reward_zero_std": 0.0, "grad_norm": 0.30118489265441895, "kl": 0.16288149892352521, "learning_rate": 6.025410727259885e-06, "loss": -0.1624, "num_tokens": 21733665.0, "reward": 0.08124999701976776, "reward_std": 0.2505381107330322, "rewards/rollout_reward_func/mean": 0.08124999701976776, "rewards/rollout_reward_func/std": 0.31667375564575195, "sampling/importance_sampling_ratio/max": 2.1540331840515137, "sampling/importance_sampling_ratio/mean": 0.45772749185562134, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.32639694213867, "sampling/sampling_logp_difference/mean": 2.2054927349090576, "step": 485, "step_time": 19.473834063042887 }, { "clip_ratio/high_max": 0.012658475898206234, "clip_ratio/high_mean": 0.006329237949103117, "clip_ratio/low_mean": 0.008604459930211306, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014933697879314423, "completions/clipped_ratio": 0.15625, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 24.1875, "completions/mean_terminated_length": 23.074073791503906, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.447660505771637, "epoch": 0.00972, "frac_reward_zero_std": 0.0, "grad_norm": 0.31648704409599304, "kl": 0.22754185227677226, "learning_rate": 6.01181706489911e-06, "loss": -0.0242, "num_tokens": 21780277.0, "reward": 0.00937499850988388, "reward_std": 0.3531803488731384, "rewards/rollout_reward_func/mean": 0.00937499850988388, "rewards/rollout_reward_func/std": 0.3550187349319458, "sampling/importance_sampling_ratio/max": 1.8685340881347656, "sampling/importance_sampling_ratio/mean": 0.21657875180244446, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.657737731933594, "sampling/sampling_logp_difference/mean": 3.039926052093506, "step": 486, "step_time": 20.13957761997881 }, { "clip_ratio/high_max": 0.007337687071412802, "clip_ratio/high_mean": 0.003668843535706401, "clip_ratio/low_mean": 0.002565681468695402, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006234525004401803, "completions/clipped_ratio": 0.125, "completions/max_length": 31.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 20.3125, "completions/mean_terminated_length": 19.85714340209961, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0945758260786533, "epoch": 0.00974, "frac_reward_zero_std": 0.0, "grad_norm": 0.17103005945682526, "kl": 0.18322988087311387, "learning_rate": 5.99822654433226e-06, "loss": -0.1366, "num_tokens": 21825865.0, "reward": 0.02812500298023224, "reward_std": 0.2879886031150818, "rewards/rollout_reward_func/mean": 0.02812500298023224, "rewards/rollout_reward_func/std": 0.27967074513435364, "sampling/importance_sampling_ratio/max": 1.353405475616455, "sampling/importance_sampling_ratio/mean": 0.2868843674659729, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.41695022583008, "sampling/sampling_logp_difference/mean": 2.2080743312835693, "step": 487, "step_time": 19.850233937002486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0026900183875113726, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026900183875113726, "completions/clipped_ratio": 0.09375, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 18.6875, "completions/mean_terminated_length": 18.034482955932617, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0725770872086287, "epoch": 0.00976, "frac_reward_zero_std": 0.0, "grad_norm": 0.45345544815063477, "kl": 0.15475548268295825, "learning_rate": 5.984639344827491e-06, "loss": -0.1416, "num_tokens": 21871394.0, "reward": 0.07187500596046448, "reward_std": 0.25808173418045044, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.26668137311935425, "sampling/importance_sampling_ratio/max": 2.306605815887451, "sampling/importance_sampling_ratio/mean": 0.49116751551628113, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.67937088012695, "sampling/sampling_logp_difference/mean": 2.292904853820801, "step": 488, "step_time": 19.675358521009912 }, { "clip_ratio/high_max": 0.008721064776182175, "clip_ratio/high_mean": 0.004360532388091087, "clip_ratio/low_mean": 0.003998316591605544, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008358848979696631, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 16.5, "completions/mean_terminated_length": 15.285715103149414, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.9462857414036989, "epoch": 0.00978, "frac_reward_zero_std": 0.0, "grad_norm": 0.36508914828300476, "kl": 0.4265559681225568, "learning_rate": 5.971055645609147e-06, "loss": -0.2465, "num_tokens": 21915837.0, "reward": -0.02187500149011612, "reward_std": 0.31392860412597656, "rewards/rollout_reward_func/mean": -0.02187500149011612, "rewards/rollout_reward_func/std": 0.3159566819667816, "sampling/importance_sampling_ratio/max": 1.900664210319519, "sampling/importance_sampling_ratio/mean": 0.5698368549346924, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.482574462890625, "sampling/sampling_logp_difference/mean": 1.761659860610962, "step": 489, "step_time": 18.802291712971055 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0018367553129792213, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018367553129792213, "completions/clipped_ratio": 0.34375, "completions/max_length": 46.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 17.8125, "completions/mean_terminated_length": 15.047618865966797, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.9414891134947538, "epoch": 0.0098, "frac_reward_zero_std": 0.0, "grad_norm": 0.3156346380710602, "kl": 0.3131605724338442, "learning_rate": 5.957475625855404e-06, "loss": -0.0683, "num_tokens": 21960094.0, "reward": 0.00937499850988388, "reward_std": 0.28791147470474243, "rewards/rollout_reward_func/mean": 0.00937499850988388, "rewards/rollout_reward_func/std": 0.3009226322174072, "sampling/importance_sampling_ratio/max": 1.787507176399231, "sampling/importance_sampling_ratio/mean": 0.43058067560195923, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.66934585571289, "sampling/sampling_logp_difference/mean": 1.796460509300232, "step": 490, "step_time": 19.205086834001122 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024519230937585235, "completions/clipped_ratio": 0.25, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 20.46875, "completions/mean_terminated_length": 20.33333396911621, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1978122871369123, "epoch": 0.00982, "frac_reward_zero_std": 0.0, "grad_norm": 0.31983596086502075, "kl": 0.2509133268613368, "learning_rate": 5.943899464695905e-06, "loss": -0.1337, "num_tokens": 22005042.0, "reward": 0.006249997764825821, "reward_std": 0.288600355386734, "rewards/rollout_reward_func/mean": 0.006249997764825821, "rewards/rollout_reward_func/std": 0.30684399604797363, "sampling/importance_sampling_ratio/max": 1.2756025791168213, "sampling/importance_sampling_ratio/mean": 0.3404991328716278, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.070037841796875, "sampling/sampling_logp_difference/mean": 2.4946529865264893, "step": 491, "step_time": 18.86438020397327 }, { "clip_ratio/high_max": 0.005965909222140908, "clip_ratio/high_mean": 0.002982954611070454, "clip_ratio/low_mean": 0.004776686662808061, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007759641273878515, "completions/clipped_ratio": 0.09375, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 21.15625, "completions/mean_terminated_length": 20.44827651977539, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.2633530832827091, "epoch": 0.00984, "frac_reward_zero_std": 0.0, "grad_norm": 0.22653096914291382, "kl": 0.20189096126705408, "learning_rate": 5.930327341209392e-06, "loss": -0.1497, "num_tokens": 22050568.0, "reward": 0.12187500298023224, "reward_std": 0.24701690673828125, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.24591417610645294, "sampling/importance_sampling_ratio/max": 1.3284810781478882, "sampling/importance_sampling_ratio/mean": 0.3228551149368286, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.89113998413086, "sampling/sampling_logp_difference/mean": 2.7306222915649414, "step": 492, "step_time": 19.456697902976885 }, { "clip_ratio/high_max": 0.0044531249441206455, "clip_ratio/high_mean": 0.0022265624720603228, "clip_ratio/low_mean": 0.008315409999340773, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010541972471401095, "completions/clipped_ratio": 0.125, "completions/max_length": 48.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.0625, "completions/mean_terminated_length": 18.35714340209961, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9705310575664043, "epoch": 0.00986, "frac_reward_zero_std": 0.0, "grad_norm": 0.42289459705352783, "kl": 0.30405329330824316, "learning_rate": 5.916759434421345e-06, "loss": -0.1908, "num_tokens": 22095515.0, "reward": -0.046875, "reward_std": 0.31375089287757874, "rewards/rollout_reward_func/mean": -0.046875, "rewards/rollout_reward_func/std": 0.31925758719444275, "sampling/importance_sampling_ratio/max": 2.074707508087158, "sampling/importance_sampling_ratio/mean": 0.4405132532119751, "sampling/importance_sampling_ratio/min": 3.5032461608120427e-44, "sampling/sampling_logp_difference/max": 33.37259292602539, "sampling/sampling_logp_difference/mean": 1.877367377281189, "step": 493, "step_time": 19.62826670598588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.001984127040486783, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001984127040486783, "completions/clipped_ratio": 0.28125, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 17.5625, "completions/mean_terminated_length": 16.869565963745117, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.9138252772390842, "epoch": 0.00988, "frac_reward_zero_std": 0.0, "grad_norm": 0.784049391746521, "kl": 0.1817811281653121, "learning_rate": 5.903195923301632e-06, "loss": 0.0013, "num_tokens": 22138819.0, "reward": 0.140625, "reward_std": 0.2704407274723053, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.2662273645401001, "sampling/importance_sampling_ratio/max": 2.299823045730591, "sampling/importance_sampling_ratio/mean": 0.6119321584701538, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.00148010253906, "sampling/sampling_logp_difference/mean": 1.762620210647583, "step": 494, "step_time": 19.038283817993943 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.004120385798159987, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0053223089198581874, "completions/clipped_ratio": 0.15625, "completions/max_length": 58.0, "completions/max_terminated_length": 58.0, "completions/mean_length": 20.25, "completions/mean_terminated_length": 19.629629135131836, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.014417065307498, "epoch": 0.0099, "frac_reward_zero_std": 0.0, "grad_norm": 0.41059672832489014, "kl": 0.30804456409532577, "learning_rate": 5.88963698676213e-06, "loss": -0.2008, "num_tokens": 22183577.0, "reward": -0.06874999403953552, "reward_std": 0.4057226777076721, "rewards/rollout_reward_func/mean": -0.06874999403953552, "rewards/rollout_reward_func/std": 0.39303410053253174, "sampling/importance_sampling_ratio/max": 1.9429731369018555, "sampling/importance_sampling_ratio/mean": 0.5215327739715576, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.170143127441406, "sampling/sampling_logp_difference/mean": 2.336475133895874, "step": 495, "step_time": 20.072643282983336 }, { "clip_ratio/high_max": 0.006861772621050477, "clip_ratio/high_mean": 0.0034308863105252385, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004680886282585561, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.21875, "completions/mean_terminated_length": 18.034482955932617, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.9207961112260818, "epoch": 0.00992, "frac_reward_zero_std": 0.0, "grad_norm": 0.27388080954551697, "kl": 0.4493172154761851, "learning_rate": 5.87608280365438e-06, "loss": -0.148, "num_tokens": 22227060.0, "reward": 0.03437500447034836, "reward_std": 0.28676944971084595, "rewards/rollout_reward_func/mean": 0.03437500447034836, "rewards/rollout_reward_func/std": 0.30754950642585754, "sampling/importance_sampling_ratio/max": 2.575826406478882, "sampling/importance_sampling_ratio/mean": 0.5636743903160095, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.914241790771484, "sampling/sampling_logp_difference/mean": 1.8685543537139893, "step": 496, "step_time": 18.53669369898853 }, { "clip_ratio/high_max": 0.013938456075266004, "clip_ratio/high_mean": 0.006969228037633002, "clip_ratio/low_mean": 0.0032314311829395592, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010200659220572561, "completions/clipped_ratio": 0.1875, "completions/max_length": 83.0, "completions/max_terminated_length": 83.0, "completions/mean_length": 22.09375, "completions/mean_terminated_length": 21.615385055541992, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.050780013203621, "epoch": 0.00994, "frac_reward_zero_std": 0.0, "grad_norm": 0.2807837724685669, "kl": 0.2223488292656839, "learning_rate": 5.862533552767218e-06, "loss": -0.1019, "num_tokens": 22272648.0, "reward": -0.0312499962747097, "reward_std": 0.3726668357849121, "rewards/rollout_reward_func/mean": -0.0312499962747097, "rewards/rollout_reward_func/std": 0.40196090936660767, "sampling/importance_sampling_ratio/max": 2.3142917156219482, "sampling/importance_sampling_ratio/mean": 0.4509401321411133, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.59868621826172, "sampling/sampling_logp_difference/mean": 2.2242038249969482, "step": 497, "step_time": 21.53420912202273 }, { "clip_ratio/high_max": 0.001953125, "clip_ratio/high_mean": 0.0009765625, "clip_ratio/low_mean": 0.0014880952658131719, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002464657765813172, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.4375, "completions/mean_terminated_length": 18.481481552124023, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9838048797100782, "epoch": 0.00996, "frac_reward_zero_std": 0.0, "grad_norm": 0.2843092083930969, "kl": 0.812431319616735, "learning_rate": 5.848989412824425e-06, "loss": 0.0266, "num_tokens": 22317800.0, "reward": 0.11562500149011612, "reward_std": 0.31124675273895264, "rewards/rollout_reward_func/mean": 0.11562500149011612, "rewards/rollout_reward_func/std": 0.3122337758541107, "sampling/importance_sampling_ratio/max": 1.2693068981170654, "sampling/importance_sampling_ratio/mean": 0.3669273853302002, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.97368621826172, "sampling/sampling_logp_difference/mean": 2.0625219345092773, "step": 498, "step_time": 18.752370270041865 }, { "clip_ratio/high_max": 0.005972222192212939, "clip_ratio/high_mean": 0.0029861110961064696, "clip_ratio/low_mean": 0.0030573620460927486, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006043473142199218, "completions/clipped_ratio": 0.15625, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 18.9375, "completions/mean_terminated_length": 19.0, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9446221627295017, "epoch": 0.00998, "frac_reward_zero_std": 0.0, "grad_norm": 0.3054559826850891, "kl": 0.1753198066726327, "learning_rate": 5.8354505624823585e-06, "loss": -0.1496, "num_tokens": 22362237.0, "reward": -0.109375, "reward_std": 0.34653526544570923, "rewards/rollout_reward_func/mean": -0.109375, "rewards/rollout_reward_func/std": 0.3476737141609192, "sampling/importance_sampling_ratio/max": 1.237502098083496, "sampling/importance_sampling_ratio/mean": 0.44608715176582336, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.81302261352539, "sampling/sampling_logp_difference/mean": 1.9945952892303467, "step": 499, "step_time": 18.85246019801707 }, { "clip_ratio/high_max": 0.001953125, "clip_ratio/high_mean": 0.0009765625, "clip_ratio/low_mean": 0.0010416667209938169, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002018229220993817, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 19.46875, "completions/mean_terminated_length": 18.653846740722656, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.001788979396224, "epoch": 0.01, "frac_reward_zero_std": 0.0, "grad_norm": 0.26429998874664307, "kl": 0.28267577453516424, "learning_rate": 5.821917180327612e-06, "loss": -0.0694, "num_tokens": 22407925.0, "reward": -0.02187499776482582, "reward_std": 0.29391759634017944, "rewards/rollout_reward_func/mean": -0.02187499776482582, "rewards/rollout_reward_func/std": 0.32402148842811584, "sampling/importance_sampling_ratio/max": 2.1744863986968994, "sampling/importance_sampling_ratio/mean": 0.4302528202533722, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.82638931274414, "sampling/sampling_logp_difference/mean": 2.067466974258423, "step": 500, "step_time": 19.776117507033632 }, { "clip_ratio/high_max": 0.007104706717655063, "clip_ratio/high_mean": 0.0035523533588275313, "clip_ratio/low_mean": 0.0014880952658131719, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005040448624640703, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 19.78125, "completions/mean_terminated_length": 19.538461685180664, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1713082268834114, "epoch": 0.01002, "frac_reward_zero_std": 0.0, "grad_norm": 0.40692609548568726, "kl": 0.47606318327598274, "learning_rate": 5.808389444874641e-06, "loss": -0.1365, "num_tokens": 22452602.0, "reward": 0.009375003166496754, "reward_std": 0.28495389223098755, "rewards/rollout_reward_func/mean": 0.009375003166496754, "rewards/rollout_reward_func/std": 0.27981486916542053, "sampling/importance_sampling_ratio/max": 2.2834548950195312, "sampling/importance_sampling_ratio/mean": 0.4234563112258911, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.628868103027344, "sampling/sampling_logp_difference/mean": 2.559474229812622, "step": 501, "step_time": 19.783145992012578 }, { "clip_ratio/high_max": 0.008623426430858672, "clip_ratio/high_mean": 0.004311713215429336, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004311713215429336, "completions/clipped_ratio": 0.1875, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 18.0, "completions/mean_terminated_length": 16.961538314819336, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9253351408988237, "epoch": 0.01004, "frac_reward_zero_std": 0.0, "grad_norm": 0.25732651352882385, "kl": 0.2543816422112286, "learning_rate": 5.794867534563422e-06, "loss": -0.0879, "num_tokens": 22496025.0, "reward": 0.1093750074505806, "reward_std": 0.29481539130210876, "rewards/rollout_reward_func/mean": 0.1093750074505806, "rewards/rollout_reward_func/std": 0.3009226322174072, "sampling/importance_sampling_ratio/max": 2.514075994491577, "sampling/importance_sampling_ratio/mean": 0.592729926109314, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.251495361328125, "sampling/sampling_logp_difference/mean": 2.0131564140319824, "step": 502, "step_time": 18.906950099990354 }, { "clip_ratio/high_max": 0.003289473708719015, "clip_ratio/high_mean": 0.0016447368543595076, "clip_ratio/low_mean": 0.004546957788988948, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006191694643348455, "completions/clipped_ratio": 0.1875, "completions/max_length": 56.0, "completions/max_terminated_length": 56.0, "completions/mean_length": 21.09375, "completions/mean_terminated_length": 21.653846740722656, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1088721938431263, "epoch": 0.01006, "frac_reward_zero_std": 0.0, "grad_norm": 0.4308432936668396, "kl": 0.6829791415948421, "learning_rate": 5.781351627757093e-06, "loss": -0.095, "num_tokens": 22541339.0, "reward": 0.02187500149011612, "reward_std": 0.33469271659851074, "rewards/rollout_reward_func/mean": 0.02187500149011612, "rewards/rollout_reward_func/std": 0.33091697096824646, "sampling/importance_sampling_ratio/max": 1.5789870023727417, "sampling/importance_sampling_ratio/mean": 0.3229069411754608, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.49827575683594, "sampling/sampling_logp_difference/mean": 2.438380002975464, "step": 503, "step_time": 19.896741324002505 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.001550099259475246, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0027520223811734468, "completions/clipped_ratio": 0.15625, "completions/max_length": 72.0, "completions/max_terminated_length": 72.0, "completions/mean_length": 19.71875, "completions/mean_terminated_length": 19.814815521240234, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.980621088296175, "epoch": 0.01008, "frac_reward_zero_std": 0.0, "grad_norm": 0.3290412724018097, "kl": 0.2988561438396573, "learning_rate": 5.767841902739602e-06, "loss": -0.1701, "num_tokens": 22585132.0, "reward": -0.0937499925494194, "reward_std": 0.3707539439201355, "rewards/rollout_reward_func/mean": -0.0937499925494194, "rewards/rollout_reward_func/std": 0.3749731183052063, "sampling/importance_sampling_ratio/max": 1.5519789457321167, "sampling/importance_sampling_ratio/mean": 0.4219784438610077, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.966339111328125, "sampling/sampling_logp_difference/mean": 1.9912981986999512, "step": 504, "step_time": 20.47152828295657 }, { "clip_ratio/high_max": 0.013902764301747084, "clip_ratio/high_mean": 0.006951382150873542, "clip_ratio/low_mean": 0.005584490834735334, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012535872752778232, "completions/clipped_ratio": 0.28125, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 21.75, "completions/mean_terminated_length": 21.782609939575195, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.2924047261476517, "epoch": 0.0101, "frac_reward_zero_std": 0.0, "grad_norm": 0.21272806823253632, "kl": 0.37129758251830935, "learning_rate": 5.754338537713353e-06, "loss": -0.1211, "num_tokens": 22630962.0, "reward": -0.003124997951090336, "reward_std": 0.35085371136665344, "rewards/rollout_reward_func/mean": -0.003124997951090336, "rewards/rollout_reward_func/std": 0.334552526473999, "sampling/importance_sampling_ratio/max": 1.4051731824874878, "sampling/importance_sampling_ratio/mean": 0.20681598782539368, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.170143127441406, "sampling/sampling_logp_difference/mean": 2.55310320854187, "step": 505, "step_time": 20.319065742995008 }, { "clip_ratio/high_max": 0.010416666744276881, "clip_ratio/high_mean": 0.005208333372138441, "clip_ratio/low_mean": 0.005074300803244114, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010282634058967233, "completions/clipped_ratio": 0.15625, "completions/max_length": 43.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 18.1875, "completions/mean_terminated_length": 17.66666603088379, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.8958708494901657, "epoch": 0.01012, "frac_reward_zero_std": 0.0, "grad_norm": 0.25478577613830566, "kl": 0.16820441861636937, "learning_rate": 5.740841710796861e-06, "loss": -0.0868, "num_tokens": 22674884.0, "reward": 0.11562499403953552, "reward_std": 0.22455009818077087, "rewards/rollout_reward_func/mean": 0.11562499403953552, "rewards/rollout_reward_func/std": 0.24111217260360718, "sampling/importance_sampling_ratio/max": 1.2570345401763916, "sampling/importance_sampling_ratio/mean": 0.43914929032325745, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.438987731933594, "sampling/sampling_logp_difference/mean": 1.956439733505249, "step": 506, "step_time": 19.198966187032056 }, { "clip_ratio/high_max": 0.0035978618543595076, "clip_ratio/high_mean": 0.0017989309271797538, "clip_ratio/low_mean": 0.0008928571478463709, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026917880750261247, "completions/clipped_ratio": 0.15625, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 19.375, "completions/mean_terminated_length": 18.22222328186035, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9127821531146765, "epoch": 0.01014, "frac_reward_zero_std": 0.0, "grad_norm": 0.43903976678848267, "kl": 0.1497212357353419, "learning_rate": 5.727351600022396e-06, "loss": -0.1463, "num_tokens": 22719250.0, "reward": 0.028125006705522537, "reward_std": 0.2647896409034729, "rewards/rollout_reward_func/mean": 0.028125006705522537, "rewards/rollout_reward_func/std": 0.26668134331703186, "sampling/importance_sampling_ratio/max": 2.3952414989471436, "sampling/importance_sampling_ratio/mean": 0.5162743330001831, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.31743621826172, "sampling/sampling_logp_difference/mean": 1.8897738456726074, "step": 507, "step_time": 19.413932126990403 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.0038734408444724977, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052938954322598875, "completions/clipped_ratio": 0.15625, "completions/max_length": 50.0, "completions/max_terminated_length": 50.0, "completions/mean_length": 22.03125, "completions/mean_terminated_length": 21.148147583007812, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.128622530028224, "epoch": 0.01016, "frac_reward_zero_std": 0.0, "grad_norm": 0.4245031177997589, "kl": 0.3278340082615614, "learning_rate": 5.7138683833336385e-06, "loss": -0.1822, "num_tokens": 22765105.0, "reward": -0.0625, "reward_std": 0.28701984882354736, "rewards/rollout_reward_func/mean": -0.0625, "rewards/rollout_reward_func/std": 0.2926519215106964, "sampling/importance_sampling_ratio/max": 1.4360737800598145, "sampling/importance_sampling_ratio/mean": 0.2867375612258911, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.466346740722656, "sampling/sampling_logp_difference/mean": 2.4126696586608887, "step": 508, "step_time": 19.915766318968963 }, { "clip_ratio/high_max": 0.0193452388048172, "clip_ratio/high_mean": 0.0096726194024086, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010922619374468923, "completions/clipped_ratio": 0.125, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 19.3125, "completions/mean_terminated_length": 18.642858505249023, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.1095628011971712, "epoch": 0.01018, "frac_reward_zero_std": 0.0, "grad_norm": 0.41105106472969055, "kl": 0.19746088655665517, "learning_rate": 5.70039223858333e-06, "loss": -0.0924, "num_tokens": 22811339.0, "reward": -0.009375001303851604, "reward_std": 0.3000386953353882, "rewards/rollout_reward_func/mean": -0.009375001303851604, "rewards/rollout_reward_func/std": 0.287771612405777, "sampling/importance_sampling_ratio/max": 2.250863552093506, "sampling/importance_sampling_ratio/mean": 0.4000692367553711, "sampling/importance_sampling_ratio/min": 4.035739577255473e-43, "sampling/sampling_logp_difference/max": 35.305702209472656, "sampling/sampling_logp_difference/mean": 2.038846969604492, "step": 509, "step_time": 20.033293866028544 }, { "clip_ratio/high_max": 0.01301393611356616, "clip_ratio/high_mean": 0.00650696805678308, "clip_ratio/low_mean": 0.0030948068015277386, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009601774741895497, "completions/clipped_ratio": 0.15625, "completions/max_length": 46.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 22.125, "completions/mean_terminated_length": 20.407407760620117, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.188265509903431, "epoch": 0.0102, "frac_reward_zero_std": 0.0, "grad_norm": 0.2452629804611206, "kl": 0.2870444286381826, "learning_rate": 5.686923343530932e-06, "loss": -0.1567, "num_tokens": 22856894.0, "reward": -0.015624996274709702, "reward_std": 0.35236573219299316, "rewards/rollout_reward_func/mean": -0.015624996274709702, "rewards/rollout_reward_func/std": 0.34276801347732544, "sampling/importance_sampling_ratio/max": 2.647519588470459, "sampling/importance_sampling_ratio/mean": 0.3548222482204437, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.20359420776367, "sampling/sampling_logp_difference/mean": 2.5800018310546875, "step": 510, "step_time": 19.645712232988444 }, { "clip_ratio/high_max": 0.005859375, "clip_ratio/high_mean": 0.0029296875, "clip_ratio/low_mean": 0.0012499999720603228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004179687472060323, "completions/clipped_ratio": 0.09375, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 18.9375, "completions/mean_terminated_length": 18.13793182373047, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0143097583204508, "epoch": 0.01022, "frac_reward_zero_std": 0.0, "grad_norm": 0.2771323323249817, "kl": 0.24957090290263295, "learning_rate": 5.673461875840275e-06, "loss": -0.0124, "num_tokens": 22902103.0, "reward": 0.04375000298023224, "reward_std": 0.22834472358226776, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.2474873811006546, "sampling/importance_sampling_ratio/max": 2.345998764038086, "sampling/importance_sampling_ratio/mean": 0.3370611071586609, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.53177261352539, "sampling/sampling_logp_difference/mean": 2.3135838508605957, "step": 511, "step_time": 18.839163416952942 }, { "clip_ratio/high_max": 0.01235702633857727, "clip_ratio/high_mean": 0.006178513169288635, "clip_ratio/low_mean": 0.0015434451051987708, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007721958274487406, "completions/clipped_ratio": 0.21875, "completions/max_length": 47.0, "completions/max_terminated_length": 47.0, "completions/mean_length": 22.6875, "completions/mean_terminated_length": 20.920000076293945, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.281884916126728, "epoch": 0.01024, "frac_reward_zero_std": 0.0, "grad_norm": 0.309592604637146, "kl": 0.16347592952661216, "learning_rate": 5.6600080130772166e-06, "loss": -0.1358, "num_tokens": 22947915.0, "reward": -0.07500000298023224, "reward_std": 0.3723684549331665, "rewards/rollout_reward_func/mean": -0.07500000298023224, "rewards/rollout_reward_func/std": 0.3645589351654053, "sampling/importance_sampling_ratio/max": 1.8058441877365112, "sampling/importance_sampling_ratio/mean": 0.27991098165512085, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.42573165893555, "sampling/sampling_logp_difference/mean": 2.7471072673797607, "step": 512, "step_time": 20.634554143965943 }, { "clip_ratio/high_max": 0.013036859221756458, "clip_ratio/high_mean": 0.006518429610878229, "clip_ratio/low_mean": 0.0022916666930541396, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00881009642034769, "completions/clipped_ratio": 0.15625, "completions/max_length": 46.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 20.875, "completions/mean_terminated_length": 19.814815521240234, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0600677207112312, "epoch": 0.01026, "frac_reward_zero_std": 0.0, "grad_norm": 0.23464275896549225, "kl": 0.15872521849814802, "learning_rate": 5.646561932707302e-06, "loss": -0.1465, "num_tokens": 22992287.0, "reward": -0.03437500447034836, "reward_std": 0.36513519287109375, "rewards/rollout_reward_func/mean": -0.03437500447034836, "rewards/rollout_reward_func/std": 0.358859658241272, "sampling/importance_sampling_ratio/max": 1.477960467338562, "sampling/importance_sampling_ratio/mean": 0.32361334562301636, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.98823928833008, "sampling/sampling_logp_difference/mean": 2.094515562057495, "step": 513, "step_time": 19.48033066598873 }, { "clip_ratio/high_max": 0.004655172349885106, "clip_ratio/high_mean": 0.002327586174942553, "clip_ratio/low_mean": 0.007311076857149601, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009638663032092154, "completions/clipped_ratio": 0.28125, "completions/max_length": 46.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 19.9375, "completions/mean_terminated_length": 17.2608699798584, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0599937178194523, "epoch": 0.01028, "frac_reward_zero_std": 0.0, "grad_norm": 0.4043598473072052, "kl": 0.3506082482635975, "learning_rate": 5.633123812093419e-06, "loss": -0.1358, "num_tokens": 23036621.0, "reward": 0.09062500298023224, "reward_std": 0.3081700801849365, "rewards/rollout_reward_func/mean": 0.09062500298023224, "rewards/rollout_reward_func/std": 0.3124919533729553, "sampling/importance_sampling_ratio/max": 2.3214781284332275, "sampling/importance_sampling_ratio/mean": 0.4362459182739258, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.75386428833008, "sampling/sampling_logp_difference/mean": 2.15004301071167, "step": 514, "step_time": 20.981058988007135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0032201523426920176, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032201523426920176, "completions/clipped_ratio": 0.1875, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 20.0625, "completions/mean_terminated_length": 18.69230842590332, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9712546579539776, "epoch": 0.0103, "frac_reward_zero_std": 0.0, "grad_norm": 0.45423424243927, "kl": 0.16981028532609344, "learning_rate": 5.619693828493467e-06, "loss": -0.246, "num_tokens": 23080022.0, "reward": 0.03125, "reward_std": 0.28020381927490234, "rewards/rollout_reward_func/mean": 0.03125, "rewards/rollout_reward_func/std": 0.2822004556655884, "sampling/importance_sampling_ratio/max": 2.6852164268493652, "sampling/importance_sampling_ratio/mean": 0.5360272526741028, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 33.161190032958984, "sampling/sampling_logp_difference/mean": 1.91825270652771, "step": 515, "step_time": 18.74659023497952 }, { "clip_ratio/high_max": 0.0024038462433964014, "clip_ratio/high_mean": 0.0012019231216982007, "clip_ratio/low_mean": 0.005673363222740591, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006875286344438791, "completions/clipped_ratio": 0.15625, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 18.71875, "completions/mean_terminated_length": 18.185184478759766, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.8601151220500469, "epoch": 0.01032, "frac_reward_zero_std": 0.0, "grad_norm": 0.2645263075828552, "kl": 0.2792705353349447, "learning_rate": 5.606272159058005e-06, "loss": -0.143, "num_tokens": 23124714.0, "reward": 0.07187500596046448, "reward_std": 0.2573243975639343, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.24917200207710266, "sampling/importance_sampling_ratio/max": 1.2799803018569946, "sampling/importance_sampling_ratio/mean": 0.4045022130012512, "sampling/importance_sampling_ratio/min": 1.401298464324817e-45, "sampling/sampling_logp_difference/max": 32.98265075683594, "sampling/sampling_logp_difference/mean": 1.7584569454193115, "step": 516, "step_time": 19.88664161800989 }, { "clip_ratio/high_max": 0.007170516299083829, "clip_ratio/high_mean": 0.0035852581495419145, "clip_ratio/low_mean": 0.00231799460016191, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005903252633288503, "completions/clipped_ratio": 0.15625, "completions/max_length": 40.0, "completions/max_terminated_length": 40.0, "completions/mean_length": 18.8125, "completions/mean_terminated_length": 18.33333396911621, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.9342909306287766, "epoch": 0.01034, "frac_reward_zero_std": 0.0, "grad_norm": 0.5361873507499695, "kl": 0.17398127261549234, "learning_rate": 5.5928589808279266e-06, "loss": -0.3401, "num_tokens": 23167825.0, "reward": 0.03750000149011612, "reward_std": 0.33121025562286377, "rewards/rollout_reward_func/mean": 0.03750000149011612, "rewards/rollout_reward_func/std": 0.3220398724079132, "sampling/importance_sampling_ratio/max": 2.873915910720825, "sampling/importance_sampling_ratio/mean": 0.6975578665733337, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.02444076538086, "sampling/sampling_logp_difference/mean": 1.9350833892822266, "step": 517, "step_time": 18.82180360997154 }, { "clip_ratio/high_max": 0.0047186610754579306, "clip_ratio/high_mean": 0.0023593305377289653, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004312455537728965, "completions/clipped_ratio": 0.3125, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 18.84375, "completions/mean_terminated_length": 18.18181800842285, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0473293084651232, "epoch": 0.01036, "frac_reward_zero_std": 0.0, "grad_norm": 0.5909927487373352, "kl": 0.7400109539739788, "learning_rate": 5.5794544707321184e-06, "loss": -0.1645, "num_tokens": 23212470.0, "reward": -0.0031249988824129105, "reward_std": 0.3622628450393677, "rewards/rollout_reward_func/mean": -0.0031249988824129105, "rewards/rollout_reward_func/std": 0.34963980317115784, "sampling/importance_sampling_ratio/max": 2.103813409805298, "sampling/importance_sampling_ratio/mean": 0.4761974811553955, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.82463455200195, "sampling/sampling_logp_difference/mean": 2.262355327606201, "step": 518, "step_time": 19.00025076403108 }, { "clip_ratio/high_max": 0.001953125, "clip_ratio/high_mean": 0.0009765625, "clip_ratio/low_mean": 0.003936557681299746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004913120181299746, "completions/clipped_ratio": 0.09375, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 18.03125, "completions/mean_terminated_length": 17.13793182373047, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.8727823551744223, "epoch": 0.01038, "frac_reward_zero_std": 0.0, "grad_norm": 0.21213054656982422, "kl": 0.17174413846805692, "learning_rate": 5.566058805585135e-06, "loss": -0.1743, "num_tokens": 23256561.0, "reward": 0.05000000447034836, "reward_std": 0.29866114258766174, "rewards/rollout_reward_func/mean": 0.05000000447034836, "rewards/rollout_reward_func/std": 0.30900463461875916, "sampling/importance_sampling_ratio/max": 2.292907238006592, "sampling/importance_sampling_ratio/mean": 0.4671085476875305, "sampling/importance_sampling_ratio/min": 2.802596928649634e-45, "sampling/sampling_logp_difference/max": 33.800689697265625, "sampling/sampling_logp_difference/mean": 2.0329508781433105, "step": 519, "step_time": 19.129862563990173 }, { "clip_ratio/high_max": 0.004814814776182175, "clip_ratio/high_mean": 0.0024074073880910873, "clip_ratio/low_mean": 0.006538035348057747, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008945442736148834, "completions/clipped_ratio": 0.21875, "completions/max_length": 38.0, "completions/max_terminated_length": 38.0, "completions/mean_length": 19.0, "completions/mean_terminated_length": 18.68000030517578, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0089607741683722, "epoch": 0.0104, "frac_reward_zero_std": 0.0, "grad_norm": 0.8883534073829651, "kl": 0.23307036305777729, "learning_rate": 5.55267216208485e-06, "loss": -0.1691, "num_tokens": 23302132.0, "reward": 0.046875, "reward_std": 0.24792805314064026, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.2929101884365082, "sampling/importance_sampling_ratio/max": 2.303149700164795, "sampling/importance_sampling_ratio/mean": 0.49667373299598694, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.99758529663086, "sampling/sampling_logp_difference/mean": 2.173562526702881, "step": 520, "step_time": 18.844162129011238 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0037280702963471413, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005681195296347141, "completions/clipped_ratio": 0.1875, "completions/max_length": 43.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 20.0, "completions/mean_terminated_length": 19.730770111083984, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0705969240516424, "epoch": 0.01042, "frac_reward_zero_std": 0.0, "grad_norm": 0.3135278820991516, "kl": 0.17820392758585513, "learning_rate": 5.539294716810144e-06, "loss": -0.1147, "num_tokens": 23347139.0, "reward": 0.04375000298023224, "reward_std": 0.30764061212539673, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.3004700541496277, "sampling/importance_sampling_ratio/max": 1.363632082939148, "sampling/importance_sampling_ratio/mean": 0.37042081356048584, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.28274154663086, "sampling/sampling_logp_difference/mean": 2.146625280380249, "step": 521, "step_time": 18.978375547987525 }, { "clip_ratio/high_max": 0.007211538730189204, "clip_ratio/high_mean": 0.004349817056208849, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004349817056208849, "completions/clipped_ratio": 0.21875, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 19.875, "completions/mean_terminated_length": 17.84000015258789, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.8941397983580828, "epoch": 0.01044, "frac_reward_zero_std": 0.0, "grad_norm": 0.19665399193763733, "kl": 0.37590601737610996, "learning_rate": 5.525926646218561e-06, "loss": -0.1238, "num_tokens": 23390133.0, "reward": 0.14375001192092896, "reward_std": 0.3015366792678833, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.2906472086906433, "sampling/importance_sampling_ratio/max": 2.2944436073303223, "sampling/importance_sampling_ratio/mean": 0.5747863054275513, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.0014762878418, "sampling/sampling_logp_difference/mean": 1.8335576057434082, "step": 522, "step_time": 18.940299138994305 }, { "clip_ratio/high_max": 0.006888692732900381, "clip_ratio/high_mean": 0.0034443463664501905, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034443463664501905, "completions/clipped_ratio": 0.34375, "completions/max_length": 32.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 20.3125, "completions/mean_terminated_length": 20.4761905670166, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.062382958829403, "epoch": 0.01046, "frac_reward_zero_std": 0.0, "grad_norm": 0.5401050448417664, "kl": 0.34021043311804533, "learning_rate": 5.512568126643991e-06, "loss": -0.1197, "num_tokens": 23435833.0, "reward": 0.0031249988824129105, "reward_std": 0.27542996406555176, "rewards/rollout_reward_func/mean": 0.0031249988824129105, "rewards/rollout_reward_func/std": 0.2717736065387726, "sampling/importance_sampling_ratio/max": 2.3066651821136475, "sampling/importance_sampling_ratio/mean": 0.3858146667480469, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.01308822631836, "sampling/sampling_logp_difference/mean": 1.7936913967132568, "step": 523, "step_time": 20.077934216998983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0013020833721384406, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013020833721384406, "completions/clipped_ratio": 0.1875, "completions/max_length": 50.0, "completions/max_terminated_length": 50.0, "completions/mean_length": 19.09375, "completions/mean_terminated_length": 17.346155166625977, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.9136356599628925, "epoch": 0.01048, "frac_reward_zero_std": 0.0, "grad_norm": 0.36146554350852966, "kl": 0.312109841266647, "learning_rate": 5.499219334294335e-06, "loss": -0.0933, "num_tokens": 23479561.0, "reward": 0.02187500149011612, "reward_std": 0.38508591055870056, "rewards/rollout_reward_func/mean": 0.02187500149011612, "rewards/rollout_reward_func/std": 0.3722073435783386, "sampling/importance_sampling_ratio/max": 2.5195138454437256, "sampling/importance_sampling_ratio/mean": 0.6317037343978882, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.89944839477539, "sampling/sampling_logp_difference/mean": 1.9061387777328491, "step": 524, "step_time": 19.482289961975766 }, { "clip_ratio/high_max": 0.008190883323550224, "clip_ratio/high_mean": 0.004095441661775112, "clip_ratio/low_mean": 0.005854208138771355, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009949649800546467, "completions/clipped_ratio": 0.125, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 20.90625, "completions/mean_terminated_length": 20.25, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.169839784502983, "epoch": 0.0105, "frac_reward_zero_std": 0.0, "grad_norm": 0.274620920419693, "kl": 0.18219675589352846, "learning_rate": 5.485880445249192e-06, "loss": -0.1241, "num_tokens": 23524106.0, "reward": 0.05312500521540642, "reward_std": 0.31063807010650635, "rewards/rollout_reward_func/mean": 0.05312500521540642, "rewards/rollout_reward_func/std": 0.314164936542511, "sampling/importance_sampling_ratio/max": 1.9712551832199097, "sampling/importance_sampling_ratio/mean": 0.3944067060947418, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.649444580078125, "sampling/sampling_logp_difference/mean": 2.4535861015319824, "step": 525, "step_time": 18.745597723987885 } ], "logging_steps": 1.0, "max_steps": 900, "num_input_tokens_seen": 23524106, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }