| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.5714285714285714, |
| "eval_steps": 500, |
| "global_step": 500, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2523.270866394043, |
| "epoch": 0.001142857142857143, |
| "grad_norm": 0.06654668599367142, |
| "kl": 0.0, |
| "lambda_div_used": 0.6164634302258492, |
| "learning_rate": 0.0, |
| "loss": -0.0258, |
| "reward": -0.14551701629534364, |
| "reward_after_mean": -0.14551701629534364, |
| "reward_after_std": 0.6225011153146625, |
| "reward_before_mean": 0.17862090840935707, |
| "reward_before_std": 0.5394803490489721, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3241379093378782, |
| "reward_change_min": -0.5330121107399464, |
| "reward_change_std": 0.19750467501580715, |
| "reward_std": 0.622501116245985, |
| "rewards/accuracy_reward": 0.2500000037252903, |
| "rewards/cosine_scaled_reward": -0.0713790925219655, |
| "step": 1 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2684.583366394043, |
| "epoch": 0.002285714285714286, |
| "grad_norm": 0.07638121396303177, |
| "kl": 0.0, |
| "lambda_div_used": 0.5910675376653671, |
| "learning_rate": 5e-08, |
| "loss": 0.001, |
| "reward": -0.07136534340679646, |
| "reward_after_mean": -0.07136534340679646, |
| "reward_after_std": 0.5026816055178642, |
| "reward_before_mean": 0.33918463438749313, |
| "reward_before_std": 0.41114553064107895, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4105500001460314, |
| "reward_change_min": -0.619081187993288, |
| "reward_change_std": 0.23483010660856962, |
| "reward_std": 0.5026816166937351, |
| "rewards/accuracy_reward": 0.29166667349636555, |
| "rewards/cosine_scaled_reward": 0.047517990693449974, |
| "step": 2 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2966.9166870117188, |
| "epoch": 0.0034285714285714284, |
| "grad_norm": 0.07826078683137894, |
| "kl": 5.805492401123047e-05, |
| "lambda_div_used": 0.5747007578611374, |
| "learning_rate": 1e-07, |
| "loss": 0.0118, |
| "reward": -0.3506452329456806, |
| "reward_after_mean": -0.3506452329456806, |
| "reward_after_std": 0.40795029513537884, |
| "reward_before_mean": -0.06986081041395664, |
| "reward_before_std": 0.3391748256981373, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.280784422531724, |
| "reward_change_min": -0.45085202157497406, |
| "reward_change_std": 0.16326917707920074, |
| "reward_std": 0.4079503044486046, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.1323608043603599, |
| "step": 3 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1377.000015258789, |
| "epoch": 0.004571428571428572, |
| "grad_norm": 0.10993114858865738, |
| "kl": 3.951042890548706e-05, |
| "lambda_div_used": 0.5788158848881721, |
| "learning_rate": 1.5e-07, |
| "loss": 0.0214, |
| "reward": -0.24558139964938164, |
| "reward_after_mean": -0.24558139964938164, |
| "reward_after_std": 0.3998244144022465, |
| "reward_before_mean": 0.06915237568318844, |
| "reward_before_std": 0.36195528600364923, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3147337753325701, |
| "reward_change_min": -0.4869973622262478, |
| "reward_change_std": 0.1927295122295618, |
| "reward_std": 0.3998244162648916, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.09751429967582226, |
| "step": 4 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3310.4583587646484, |
| "epoch": 0.005714285714285714, |
| "grad_norm": 0.06377790123224258, |
| "kl": 5.4255127906799316e-05, |
| "lambda_div_used": 0.61885916441679, |
| "learning_rate": 2e-07, |
| "loss": 0.0428, |
| "reward": -0.25358792673796415, |
| "reward_after_mean": -0.25358792673796415, |
| "reward_after_std": 0.5858059301972389, |
| "reward_before_mean": 0.0003254720941185951, |
| "reward_before_std": 0.5446656532585621, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25391339510679245, |
| "reward_change_min": -0.45586229115724564, |
| "reward_change_std": 0.16313681472092867, |
| "reward_std": 0.5858059376478195, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.12467453628778458, |
| "step": 5 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2887.8541717529297, |
| "epoch": 0.006857142857142857, |
| "grad_norm": 0.06263996660709381, |
| "kl": 5.511939525604248e-05, |
| "lambda_div_used": 0.5606320649385452, |
| "learning_rate": 2.5e-07, |
| "loss": 0.0618, |
| "reward": -0.4456054698675871, |
| "reward_after_mean": -0.4456054698675871, |
| "reward_after_std": 0.3509902711957693, |
| "reward_before_mean": -0.1859958479180932, |
| "reward_before_std": 0.2702419590204954, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2596096061170101, |
| "reward_change_min": -0.38109203428030014, |
| "reward_change_std": 0.14125030301511288, |
| "reward_std": 0.3509902749210596, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2276625158265233, |
| "step": 6 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2572.3542098999023, |
| "epoch": 0.008, |
| "grad_norm": 0.06695970892906189, |
| "kl": 4.0218234062194824e-05, |
| "lambda_div_used": 0.6162339821457863, |
| "learning_rate": 3e-07, |
| "loss": 0.0453, |
| "reward": -0.10290078446269035, |
| "reward_after_mean": -0.10290078446269035, |
| "reward_after_std": 0.566169198602438, |
| "reward_before_mean": 0.22045390354469419, |
| "reward_before_std": 0.5333282891660929, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.32335469499230385, |
| "reward_change_min": -0.5540991388261318, |
| "reward_change_std": 0.20453903079032898, |
| "reward_std": 0.566169211640954, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": 0.012120556086301804, |
| "step": 7 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2173.7500228881836, |
| "epoch": 0.009142857142857144, |
| "grad_norm": 0.07779484242200851, |
| "kl": 3.185681998729706e-05, |
| "lambda_div_used": 0.63105358928442, |
| "learning_rate": 3.5e-07, |
| "loss": 0.0071, |
| "reward": 0.09723488846793771, |
| "reward_after_mean": 0.09723488846793771, |
| "reward_after_std": 0.6893859151750803, |
| "reward_before_mean": 0.5079077246482484, |
| "reward_before_std": 0.6077435212209821, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.410672839730978, |
| "reward_change_min": -0.638349361717701, |
| "reward_change_std": 0.2494220733642578, |
| "reward_std": 0.6893859468400478, |
| "rewards/accuracy_reward": 0.3750000074505806, |
| "rewards/cosine_scaled_reward": 0.1329077403061092, |
| "step": 8 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2828.229179382324, |
| "epoch": 0.010285714285714285, |
| "grad_norm": 0.09342571347951889, |
| "kl": 4.9620866775512695e-05, |
| "lambda_div_used": 0.6150719821453094, |
| "learning_rate": 4e-07, |
| "loss": -0.0132, |
| "reward": -0.18431122601032257, |
| "reward_after_mean": -0.18431122601032257, |
| "reward_after_std": 0.5804070886224508, |
| "reward_before_mean": 0.09753156686201692, |
| "reward_before_std": 0.5302032623440027, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2818427961319685, |
| "reward_change_min": -0.46669338643550873, |
| "reward_change_std": 0.18001185078173876, |
| "reward_std": 0.5804071053862572, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.0691350968554616, |
| "step": 9 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2469.7083587646484, |
| "epoch": 0.011428571428571429, |
| "grad_norm": 0.09024360775947571, |
| "kl": 3.8331374526023865e-05, |
| "lambda_div_used": 0.63355503231287, |
| "learning_rate": 4.5e-07, |
| "loss": -0.001, |
| "reward": -0.04430920258164406, |
| "reward_after_mean": -0.04430920258164406, |
| "reward_after_std": 0.6170141901820898, |
| "reward_before_mean": 0.2625937759876251, |
| "reward_before_std": 0.6259458847343922, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3069029748439789, |
| "reward_change_min": -0.5734602250158787, |
| "reward_change_std": 0.22167105227708817, |
| "reward_std": 0.6170141994953156, |
| "rewards/accuracy_reward": 0.2500000074505806, |
| "rewards/cosine_scaled_reward": 0.012593759223818779, |
| "step": 10 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3367.500015258789, |
| "epoch": 0.012571428571428572, |
| "grad_norm": 0.05533275753259659, |
| "kl": 5.245208740234375e-05, |
| "lambda_div_used": 0.5749180987477303, |
| "learning_rate": 5e-07, |
| "loss": -0.0467, |
| "reward": -0.35878527723252773, |
| "reward_after_mean": -0.35878527723252773, |
| "reward_after_std": 0.4030665699392557, |
| "reward_before_mean": -0.09556051343679428, |
| "reward_before_std": 0.3429161449894309, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2632247470319271, |
| "reward_change_min": -0.4096112735569477, |
| "reward_change_std": 0.1540710162371397, |
| "reward_std": 0.40306657925248146, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.19972719065845013, |
| "step": 11 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2016.958351135254, |
| "epoch": 0.013714285714285714, |
| "grad_norm": 0.08992662280797958, |
| "kl": 3.9637088775634766e-05, |
| "lambda_div_used": 0.6247389540076256, |
| "learning_rate": 5.5e-07, |
| "loss": 0.0429, |
| "reward": -0.10162727534770966, |
| "reward_after_mean": -0.10162727534770966, |
| "reward_after_std": 0.594907833263278, |
| "reward_before_mean": 0.19102710485458374, |
| "reward_before_std": 0.5805315412580967, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.292654387652874, |
| "reward_change_min": -0.5117304362356663, |
| "reward_change_std": 0.20085694547742605, |
| "reward_std": 0.5949078388512135, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": -0.03813957702368498, |
| "step": 12 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2905.0833587646484, |
| "epoch": 0.014857142857142857, |
| "grad_norm": 0.0641385093331337, |
| "kl": 4.194676876068115e-05, |
| "lambda_div_used": 0.5766140297055244, |
| "learning_rate": 6e-07, |
| "loss": 0.0437, |
| "reward": -0.3046809285879135, |
| "reward_after_mean": -0.3046809285879135, |
| "reward_after_std": 0.399059085175395, |
| "reward_before_mean": -0.018187658861279488, |
| "reward_before_std": 0.35123275220394135, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28649328649044037, |
| "reward_change_min": -0.4807186797261238, |
| "reward_change_std": 0.17643271200358868, |
| "reward_std": 0.399059085175395, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.1431876514106989, |
| "step": 13 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2520.312530517578, |
| "epoch": 0.016, |
| "grad_norm": 0.07475250959396362, |
| "kl": 3.432855010032654e-05, |
| "lambda_div_used": 0.5991310104727745, |
| "learning_rate": 6.5e-07, |
| "loss": -0.0332, |
| "reward": -0.22936711832880974, |
| "reward_after_mean": -0.22936711832880974, |
| "reward_after_std": 0.5019008349627256, |
| "reward_before_mean": 0.04907496925443411, |
| "reward_before_std": 0.4585955021902919, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27844210527837276, |
| "reward_change_min": -0.4685846194624901, |
| "reward_change_std": 0.17394172679632902, |
| "reward_std": 0.5019008629024029, |
| "rewards/accuracy_reward": 0.14583333767950535, |
| "rewards/cosine_scaled_reward": -0.09675835724920034, |
| "step": 14 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2717.0416984558105, |
| "epoch": 0.017142857142857144, |
| "grad_norm": 0.08986662328243256, |
| "kl": 4.3585896492004395e-05, |
| "lambda_div_used": 0.5765259638428688, |
| "learning_rate": 7e-07, |
| "loss": 0.0355, |
| "reward": -0.16787780448794365, |
| "reward_after_mean": -0.16787780448794365, |
| "reward_after_std": 0.4601633083075285, |
| "reward_before_mean": 0.22437161579728127, |
| "reward_before_std": 0.3496173685416579, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3922494389116764, |
| "reward_change_min": -0.5602193549275398, |
| "reward_change_std": 0.2197399353608489, |
| "reward_std": 0.46016331762075424, |
| "rewards/accuracy_reward": 0.20833333395421505, |
| "rewards/cosine_scaled_reward": 0.016038289293646812, |
| "step": 15 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3395.1875, |
| "epoch": 0.018285714285714287, |
| "grad_norm": 0.05402011796832085, |
| "kl": 4.659593105316162e-05, |
| "lambda_div_used": 0.5638558194041252, |
| "learning_rate": 7.5e-07, |
| "loss": 0.032, |
| "reward": -0.432980858720839, |
| "reward_after_mean": -0.432980858720839, |
| "reward_after_std": 0.3415633924305439, |
| "reward_before_mean": -0.16444075386971235, |
| "reward_before_std": 0.2898183651268482, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26854010485112667, |
| "reward_change_min": -0.45487307757139206, |
| "reward_change_std": 0.16323526203632355, |
| "reward_std": 0.34156340546905994, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.20610742270946503, |
| "step": 16 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2164.020854949951, |
| "epoch": 0.019428571428571427, |
| "grad_norm": 0.1185457855463028, |
| "kl": 4.2147934436798096e-05, |
| "lambda_div_used": 0.5763917565345764, |
| "learning_rate": 8e-07, |
| "loss": 0.0302, |
| "reward": -0.2335935328155756, |
| "reward_after_mean": -0.2335935328155756, |
| "reward_after_std": 0.4716028142720461, |
| "reward_before_mean": 0.13111361488699913, |
| "reward_before_std": 0.3423871146515012, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36470715142786503, |
| "reward_change_min": -0.5119686089456081, |
| "reward_change_std": 0.19370271731168032, |
| "reward_std": 0.471602825447917, |
| "rewards/accuracy_reward": 0.18750000186264515, |
| "rewards/cosine_scaled_reward": -0.05638639978133142, |
| "step": 17 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3079.4583740234375, |
| "epoch": 0.02057142857142857, |
| "grad_norm": 0.05137661099433899, |
| "kl": 3.637373447418213e-05, |
| "lambda_div_used": 0.6073044687509537, |
| "learning_rate": 8.499999999999999e-07, |
| "loss": 0.0478, |
| "reward": -0.057250723242759705, |
| "reward_after_mean": -0.057250723242759705, |
| "reward_after_std": 0.5715843215584755, |
| "reward_before_mean": 0.3262214660644531, |
| "reward_before_std": 0.5018989769741893, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.38347217813134193, |
| "reward_change_min": -0.6260374560952187, |
| "reward_change_std": 0.24400013033300638, |
| "reward_std": 0.571584340184927, |
| "rewards/accuracy_reward": 0.2916666679084301, |
| "rewards/cosine_scaled_reward": 0.034554785932414234, |
| "step": 18 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2889.8542289733887, |
| "epoch": 0.021714285714285714, |
| "grad_norm": 0.07495337724685669, |
| "kl": 4.096329212188721e-05, |
| "lambda_div_used": 0.6219106838107109, |
| "learning_rate": 9e-07, |
| "loss": 0.036, |
| "reward": 0.1660416293889284, |
| "reward_after_mean": 0.1660416293889284, |
| "reward_after_std": 0.6783627849072218, |
| "reward_before_mean": 0.6544073540717363, |
| "reward_before_std": 0.5640754774212837, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.488365713506937, |
| "reward_change_min": -0.758969146758318, |
| "reward_change_std": 0.2900321548804641, |
| "reward_std": 0.6783628333359957, |
| "rewards/accuracy_reward": 0.45833334140479565, |
| "rewards/cosine_scaled_reward": 0.19607400865061209, |
| "step": 19 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2114.8333587646484, |
| "epoch": 0.022857142857142857, |
| "grad_norm": 0.0912039652466774, |
| "kl": 2.7514994144439697e-05, |
| "lambda_div_used": 0.6171177923679352, |
| "learning_rate": 9.499999999999999e-07, |
| "loss": -0.0338, |
| "reward": -0.12753646727651358, |
| "reward_after_mean": -0.12753646727651358, |
| "reward_after_std": 0.6479649767279625, |
| "reward_before_mean": 0.2073977841064334, |
| "reward_before_std": 0.5446468549780548, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.33493425138294697, |
| "reward_change_min": -0.5443090125918388, |
| "reward_change_std": 0.20240973494946957, |
| "reward_std": 0.6479649972170591, |
| "rewards/accuracy_reward": 0.2500000037252903, |
| "rewards/cosine_scaled_reward": -0.0426022089086473, |
| "step": 20 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2578.3125, |
| "epoch": 0.024, |
| "grad_norm": 0.1021808609366417, |
| "kl": 4.0903687477111816e-05, |
| "lambda_div_used": 0.5696183741092682, |
| "learning_rate": 1e-06, |
| "loss": 0.0243, |
| "reward": -0.16086186096072197, |
| "reward_after_mean": -0.16086186096072197, |
| "reward_after_std": 0.4245347697287798, |
| "reward_before_mean": 0.23232585442019626, |
| "reward_before_std": 0.3138282438740134, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3931876849383116, |
| "reward_change_min": -0.5524330623447895, |
| "reward_change_std": 0.2125744568184018, |
| "reward_std": 0.424534784629941, |
| "rewards/accuracy_reward": 0.25000000558793545, |
| "rewards/cosine_scaled_reward": -0.01767415925860405, |
| "step": 21 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1644.437572479248, |
| "epoch": 0.025142857142857144, |
| "grad_norm": 0.11418648809194565, |
| "kl": 2.1685846149921417e-05, |
| "lambda_div_used": 0.5976943448185921, |
| "learning_rate": 9.99931462820376e-07, |
| "loss": -0.0304, |
| "reward": -0.10828415304422379, |
| "reward_after_mean": -0.10828415304422379, |
| "reward_after_std": 0.5476059578359127, |
| "reward_before_mean": 0.2575272931717336, |
| "reward_before_std": 0.4531834872905165, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36581144109368324, |
| "reward_change_min": -0.5498910807073116, |
| "reward_change_std": 0.21405170671641827, |
| "reward_std": 0.5476059839129448, |
| "rewards/accuracy_reward": 0.29166667349636555, |
| "rewards/cosine_scaled_reward": -0.03413938358426094, |
| "step": 22 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2188.687530517578, |
| "epoch": 0.026285714285714287, |
| "grad_norm": 0.0898994579911232, |
| "kl": 3.7088990211486816e-05, |
| "lambda_div_used": 0.6402537003159523, |
| "learning_rate": 9.997258721585931e-07, |
| "loss": -0.0037, |
| "reward": -0.13877022732049227, |
| "reward_after_mean": -0.13877022732049227, |
| "reward_after_std": 0.6969181355088949, |
| "reward_before_mean": 0.11387635703431442, |
| "reward_before_std": 0.6502415342256427, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2526465691626072, |
| "reward_change_min": -0.45175985619425774, |
| "reward_change_std": 0.1627071350812912, |
| "reward_std": 0.6969181522727013, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/cosine_scaled_reward": -0.05279031861573458, |
| "step": 23 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2394.3958740234375, |
| "epoch": 0.027428571428571427, |
| "grad_norm": 0.09361977875232697, |
| "kl": 2.0720064640045166e-05, |
| "lambda_div_used": 0.6309552267193794, |
| "learning_rate": 9.993832906395582e-07, |
| "loss": 0.0666, |
| "reward": 0.0798279655573424, |
| "reward_after_mean": 0.0798279655573424, |
| "reward_after_std": 0.6638341955840588, |
| "reward_before_mean": 0.4744788520038128, |
| "reward_before_std": 0.6099728401750326, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3946509025990963, |
| "reward_change_min": -0.6427228525280952, |
| "reward_change_std": 0.25018193013966084, |
| "reward_std": 0.6638342067599297, |
| "rewards/accuracy_reward": 0.35416667349636555, |
| "rewards/cosine_scaled_reward": 0.12031219294294715, |
| "step": 24 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2358.583366394043, |
| "epoch": 0.02857142857142857, |
| "grad_norm": 0.07725252956151962, |
| "kl": 4.092603921890259e-05, |
| "lambda_div_used": 0.6378427669405937, |
| "learning_rate": 9.989038226169207e-07, |
| "loss": -0.0557, |
| "reward": -0.15363326482474804, |
| "reward_after_mean": -0.15363326482474804, |
| "reward_after_std": 0.6477887704968452, |
| "reward_before_mean": 0.10259643197059631, |
| "reward_before_std": 0.6456613540649414, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25622970052063465, |
| "reward_change_min": -0.5558900497853756, |
| "reward_change_std": 0.19845533184707165, |
| "reward_std": 0.6477888077497482, |
| "rewards/accuracy_reward": 0.1875000037252903, |
| "rewards/cosine_scaled_reward": -0.08490357082337141, |
| "step": 25 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2850.937530517578, |
| "epoch": 0.029714285714285714, |
| "grad_norm": 0.06806578487157822, |
| "kl": 3.5643577575683594e-05, |
| "lambda_div_used": 0.5781113430857658, |
| "learning_rate": 9.982876141412855e-07, |
| "loss": 0.005, |
| "reward": -0.25517464708536863, |
| "reward_after_mean": -0.25517464708536863, |
| "reward_after_std": 0.47686139307916164, |
| "reward_before_mean": 0.09097394905984402, |
| "reward_before_std": 0.3564971052110195, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3461485952138901, |
| "reward_change_min": -0.4805891402065754, |
| "reward_change_std": 0.18107541371136904, |
| "reward_std": 0.476861409842968, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.07569271977990866, |
| "step": 26 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2795.3750534057617, |
| "epoch": 0.030857142857142857, |
| "grad_norm": 0.0670829713344574, |
| "kl": 5.0067901611328125e-05, |
| "lambda_div_used": 0.5909813195466995, |
| "learning_rate": 9.975348529157229e-07, |
| "loss": 0.038, |
| "reward": -0.30352935567498207, |
| "reward_after_mean": -0.30352935567498207, |
| "reward_after_std": 0.4782011900097132, |
| "reward_before_mean": -0.04106062464416027, |
| "reward_before_std": 0.42168071679770947, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2624687273055315, |
| "reward_change_min": -0.441840048879385, |
| "reward_change_std": 0.16586182732135057, |
| "reward_std": 0.47820119373500347, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.16606062231585383, |
| "step": 27 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2662.562530517578, |
| "epoch": 0.032, |
| "grad_norm": 0.07541470974683762, |
| "kl": 4.6879053115844727e-05, |
| "lambda_div_used": 0.6455363035202026, |
| "learning_rate": 9.96645768238595e-07, |
| "loss": -0.0548, |
| "reward": -0.01765006221830845, |
| "reward_after_mean": -0.01765006221830845, |
| "reward_after_std": 0.7017391249537468, |
| "reward_before_mean": 0.26933106034994125, |
| "reward_before_std": 0.6768321208655834, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2869811188429594, |
| "reward_change_min": -0.5125514604151249, |
| "reward_change_std": 0.19719962775707245, |
| "reward_std": 0.7017391547560692, |
| "rewards/accuracy_reward": 0.29166667349636555, |
| "rewards/cosine_scaled_reward": -0.022335614077746868, |
| "step": 28 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2988.1041717529297, |
| "epoch": 0.03314285714285714, |
| "grad_norm": 0.07968341559171677, |
| "kl": 4.7832727432250977e-05, |
| "lambda_div_used": 0.5851128473877907, |
| "learning_rate": 9.956206309337066e-07, |
| "loss": -0.1054, |
| "reward": -0.38509707152843475, |
| "reward_after_mean": -0.38509707152843475, |
| "reward_after_std": 0.4501145612448454, |
| "reward_before_mean": -0.14269733056426048, |
| "reward_before_std": 0.3910446595400572, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24239975214004517, |
| "reward_change_min": -0.4103321433067322, |
| "reward_change_std": 0.1474100910127163, |
| "reward_std": 0.4501145798712969, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.20519732707180083, |
| "step": 29 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2777.1042404174805, |
| "epoch": 0.03428571428571429, |
| "grad_norm": 0.0785035490989685, |
| "kl": 4.0784478187561035e-05, |
| "lambda_div_used": 0.616574227809906, |
| "learning_rate": 9.944597532678119e-07, |
| "loss": 0.0306, |
| "reward": 0.014547215774655342, |
| "reward_after_mean": 0.014547215774655342, |
| "reward_after_std": 0.6317574586719275, |
| "reward_before_mean": 0.40835407795384526, |
| "reward_before_std": 0.5376694360747933, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3938068598508835, |
| "reward_change_min": -0.5825920477509499, |
| "reward_change_std": 0.22909015510231256, |
| "reward_std": 0.631757466122508, |
| "rewards/accuracy_reward": 0.3125000037252903, |
| "rewards/cosine_scaled_reward": 0.09585406119003892, |
| "step": 30 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3100.9166870117188, |
| "epoch": 0.03542857142857143, |
| "grad_norm": 0.05876823514699936, |
| "kl": 5.622208118438721e-05, |
| "lambda_div_used": 0.5746422484517097, |
| "learning_rate": 9.931634888554935e-07, |
| "loss": -0.0301, |
| "reward": -0.373639321886003, |
| "reward_after_mean": -0.373639321886003, |
| "reward_after_std": 0.40832165256142616, |
| "reward_before_mean": -0.11681870371103287, |
| "reward_before_std": 0.3402246618643403, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2568206209689379, |
| "reward_change_min": -0.4000260457396507, |
| "reward_change_std": 0.149553001858294, |
| "reward_std": 0.4083216693252325, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.20015203207731247, |
| "step": 31 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2879.625030517578, |
| "epoch": 0.036571428571428574, |
| "grad_norm": 0.060988157987594604, |
| "kl": 4.863739013671875e-05, |
| "lambda_div_used": 0.583471342921257, |
| "learning_rate": 9.917322325514487e-07, |
| "loss": 0.0143, |
| "reward": -0.09911171346902847, |
| "reward_after_mean": -0.09911171346902847, |
| "reward_after_std": 0.46612042374908924, |
| "reward_before_mean": 0.2912287414073944, |
| "reward_before_std": 0.3850706424564123, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.39034045673906803, |
| "reward_change_min": -0.5949138030409813, |
| "reward_change_std": 0.23070038296282291, |
| "reward_std": 0.46612043119966984, |
| "rewards/accuracy_reward": 0.2916666716337204, |
| "rewards/cosine_scaled_reward": -0.0004379265010356903, |
| "step": 32 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3136.750030517578, |
| "epoch": 0.037714285714285714, |
| "grad_norm": 0.07551854848861694, |
| "kl": 4.787743091583252e-05, |
| "lambda_div_used": 0.6242911070585251, |
| "learning_rate": 9.901664203302124e-07, |
| "loss": -0.0446, |
| "reward": -0.06627794913947582, |
| "reward_after_mean": -0.06627794913947582, |
| "reward_after_std": 0.5915219262242317, |
| "reward_before_mean": 0.24627447500824928, |
| "reward_before_std": 0.5734489392489195, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3125524502247572, |
| "reward_change_min": -0.5326853170990944, |
| "reward_change_std": 0.2051102453842759, |
| "reward_std": 0.5915219560265541, |
| "rewards/accuracy_reward": 0.25000000931322575, |
| "rewards/cosine_scaled_reward": -0.0037255147472023964, |
| "step": 33 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2379.541702270508, |
| "epoch": 0.038857142857142854, |
| "grad_norm": 0.07498405873775482, |
| "kl": 4.60892915725708e-05, |
| "lambda_div_used": 0.6475758850574493, |
| "learning_rate": 9.88466529153356e-07, |
| "loss": 0.0814, |
| "reward": 0.054736172780394554, |
| "reward_after_mean": 0.054736172780394554, |
| "reward_after_std": 0.7529329154640436, |
| "reward_before_mean": 0.4133173357695341, |
| "reward_before_std": 0.6894109938293695, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3585811499506235, |
| "reward_change_min": -0.6234904117882252, |
| "reward_change_std": 0.23517589084804058, |
| "reward_std": 0.7529329396784306, |
| "rewards/accuracy_reward": 0.3333333358168602, |
| "rewards/cosine_scaled_reward": 0.07998399529606104, |
| "step": 34 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3113.1875610351562, |
| "epoch": 0.04, |
| "grad_norm": 0.1044503003358841, |
| "kl": 5.8710575103759766e-05, |
| "lambda_div_used": 0.5754920393228531, |
| "learning_rate": 9.866330768241983e-07, |
| "loss": -0.0058, |
| "reward": -0.36803475581109524, |
| "reward_after_mean": -0.36803475581109524, |
| "reward_after_std": 0.4120363052934408, |
| "reward_before_mean": -0.1010703444480896, |
| "reward_before_std": 0.3434353759512305, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2669644057750702, |
| "reward_change_min": -0.42853060737252235, |
| "reward_change_std": 0.1540099997073412, |
| "reward_std": 0.41203631460666656, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.16357035003602505, |
| "step": 35 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3350.1666870117188, |
| "epoch": 0.04114285714285714, |
| "grad_norm": 0.07969332486391068, |
| "kl": 6.110966205596924e-05, |
| "lambda_div_used": 0.5744208693504333, |
| "learning_rate": 9.846666218300807e-07, |
| "loss": -0.0514, |
| "reward": -0.4380789175629616, |
| "reward_after_mean": -0.4380789175629616, |
| "reward_after_std": 0.41310197673738003, |
| "reward_before_mean": -0.20339234871789813, |
| "reward_before_std": 0.337322598323226, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.23468656465411186, |
| "reward_change_min": -0.3448420464992523, |
| "reward_change_std": 0.12617942783981562, |
| "reward_std": 0.4131019860506058, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.24505900964140892, |
| "step": 36 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3394.3958435058594, |
| "epoch": 0.04228571428571429, |
| "grad_norm": 0.05401439592242241, |
| "kl": 4.9054622650146484e-05, |
| "lambda_div_used": 0.5599935948848724, |
| "learning_rate": 9.825677631722435e-07, |
| "loss": 0.0086, |
| "reward": -0.3875775597989559, |
| "reward_after_mean": -0.3875775597989559, |
| "reward_after_std": 0.32731432281434536, |
| "reward_before_mean": -0.10551332868635654, |
| "reward_before_std": 0.2694389373064041, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28206423483788967, |
| "reward_change_min": -0.4370834305882454, |
| "reward_change_std": 0.16284553799778223, |
| "reward_std": 0.3273143321275711, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.1888466626405716, |
| "step": 37 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3311.0833435058594, |
| "epoch": 0.04342857142857143, |
| "grad_norm": 0.05246276035904884, |
| "kl": 4.945695400238037e-05, |
| "lambda_div_used": 0.5770035535097122, |
| "learning_rate": 9.80337140183366e-07, |
| "loss": -0.0117, |
| "reward": -0.3118477761745453, |
| "reward_after_mean": -0.3118477761745453, |
| "reward_after_std": 0.38849758356809616, |
| "reward_before_mean": -0.021045896457508206, |
| "reward_before_std": 0.3480216721072793, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2908018734306097, |
| "reward_change_min": -0.44894198328256607, |
| "reward_change_std": 0.17141664400696754, |
| "reward_std": 0.3884976040571928, |
| "rewards/accuracy_reward": 0.12500000558793545, |
| "rewards/cosine_scaled_reward": -0.14604590460658073, |
| "step": 38 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2919.604179382324, |
| "epoch": 0.044571428571428574, |
| "grad_norm": 0.08766351640224457, |
| "kl": 3.2648444175720215e-05, |
| "lambda_div_used": 0.5717808604240417, |
| "learning_rate": 9.779754323328192e-07, |
| "loss": 0.032, |
| "reward": -0.14589639008045197, |
| "reward_after_mean": -0.14589639008045197, |
| "reward_after_std": 0.4250806160271168, |
| "reward_before_mean": 0.26390516571700573, |
| "reward_before_std": 0.3228347860276699, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.40980157628655434, |
| "reward_change_min": -0.5692479386925697, |
| "reward_change_std": 0.22337355464696884, |
| "reward_std": 0.42508063092827797, |
| "rewards/accuracy_reward": 0.25000000558793545, |
| "rewards/cosine_scaled_reward": 0.013905153144150972, |
| "step": 39 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2468.7916870117188, |
| "epoch": 0.045714285714285714, |
| "grad_norm": 0.07357986271381378, |
| "kl": 4.18052077293396e-05, |
| "lambda_div_used": 0.5706272348761559, |
| "learning_rate": 9.754833590196926e-07, |
| "loss": 0.0148, |
| "reward": -0.2578982161357999, |
| "reward_after_mean": -0.2578982161357999, |
| "reward_after_std": 0.4464099854230881, |
| "reward_before_mean": 0.1052470114082098, |
| "reward_before_std": 0.31700289947912097, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3631452303379774, |
| "reward_change_min": -0.5201860442757607, |
| "reward_change_std": 0.1936064399778843, |
| "reward_std": 0.4464099947363138, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.06141966814175248, |
| "step": 40 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3203.312515258789, |
| "epoch": 0.046857142857142854, |
| "grad_norm": 0.05290238931775093, |
| "kl": 5.0440430641174316e-05, |
| "lambda_div_used": 0.6480180770158768, |
| "learning_rate": 9.728616793536587e-07, |
| "loss": -0.0199, |
| "reward": -0.04493038635700941, |
| "reward_after_mean": -0.04493038635700941, |
| "reward_after_std": 0.7092454191297293, |
| "reward_before_mean": 0.23497815802693367, |
| "reward_before_std": 0.693053056485951, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2799085471779108, |
| "reward_change_min": -0.5064845345914364, |
| "reward_change_std": 0.19558908697217703, |
| "reward_std": 0.7092454340308905, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/cosine_scaled_reward": 0.005811510724015534, |
| "step": 41 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2890.9791927337646, |
| "epoch": 0.048, |
| "grad_norm": 0.09494666010141373, |
| "kl": 5.85019588470459e-05, |
| "lambda_div_used": 0.5329261645674706, |
| "learning_rate": 9.701111919237408e-07, |
| "loss": 0.0335, |
| "reward": -0.5650591850280762, |
| "reward_after_mean": -0.5650591850280762, |
| "reward_after_std": 0.225538931787014, |
| "reward_before_mean": -0.32472414150834084, |
| "reward_before_std": 0.14451794046908617, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24033503979444504, |
| "reward_change_min": -0.358134388923645, |
| "reward_change_std": 0.12634030357003212, |
| "reward_std": 0.2255389392375946, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.32472414895892143, |
| "step": 42 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3027.9583587646484, |
| "epoch": 0.04914285714285714, |
| "grad_norm": 0.05899634212255478, |
| "kl": 4.252791404724121e-05, |
| "lambda_div_used": 0.595756284892559, |
| "learning_rate": 9.672327345550543e-07, |
| "loss": -0.017, |
| "reward": -0.31989429891109467, |
| "reward_after_mean": -0.31989429891109467, |
| "reward_after_std": 0.5033334400504827, |
| "reward_before_mean": -0.07388713955879211, |
| "reward_before_std": 0.43908379040658474, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24600715190172195, |
| "reward_change_min": -0.4015818126499653, |
| "reward_change_std": 0.143897395581007, |
| "reward_std": 0.5033334512263536, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.1780538223683834, |
| "step": 43 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2372.5833587646484, |
| "epoch": 0.05028571428571429, |
| "grad_norm": 0.08952542394399643, |
| "kl": 4.1332095861434937e-05, |
| "lambda_div_used": 0.5768283307552338, |
| "learning_rate": 9.64227184053598e-07, |
| "loss": 0.0238, |
| "reward": -0.09475147165358067, |
| "reward_after_mean": -0.09475147165358067, |
| "reward_after_std": 0.4600023180246353, |
| "reward_before_mean": 0.33489263616502285, |
| "reward_before_std": 0.35124383692163974, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4296441040933132, |
| "reward_change_min": -0.6358921378850937, |
| "reward_change_std": 0.24556122440844774, |
| "reward_std": 0.4600023254752159, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": 0.06405930174514651, |
| "step": 44 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3178.125, |
| "epoch": 0.05142857142857143, |
| "grad_norm": 0.05793697386980057, |
| "kl": 4.953145980834961e-05, |
| "lambda_div_used": 0.6092750132083893, |
| "learning_rate": 9.610954559391704e-07, |
| "loss": 0.0375, |
| "reward": -0.24964951165020466, |
| "reward_after_mean": -0.24964951165020466, |
| "reward_after_std": 0.575963044539094, |
| "reward_before_mean": 0.010031561367213726, |
| "reward_before_std": 0.5040898034349084, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25968106649816036, |
| "reward_change_min": -0.41585399955511093, |
| "reward_change_std": 0.15605208091437817, |
| "reward_std": 0.5759630724787712, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.114968441426754, |
| "step": 45 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3160.895866394043, |
| "epoch": 0.052571428571428575, |
| "grad_norm": 0.06750793755054474, |
| "kl": 4.884600639343262e-05, |
| "lambda_div_used": 0.5543409436941147, |
| "learning_rate": 9.578385041664925e-07, |
| "loss": 0.0442, |
| "reward": -0.47163831628859043, |
| "reward_after_mean": -0.47163831628859043, |
| "reward_after_std": 0.32501144520938396, |
| "reward_before_mean": -0.21356616588309407, |
| "reward_before_std": 0.24316862598061562, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25807216577231884, |
| "reward_change_min": -0.3883417770266533, |
| "reward_change_std": 0.13798328768461943, |
| "reward_std": 0.3250114619731903, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.23439949192106724, |
| "step": 46 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2624.270881652832, |
| "epoch": 0.053714285714285714, |
| "grad_norm": 0.10712098330259323, |
| "kl": 5.3569674491882324e-05, |
| "lambda_div_used": 0.6219898834824562, |
| "learning_rate": 9.54457320834625e-07, |
| "loss": 0.0102, |
| "reward": -0.10781855694949627, |
| "reward_after_mean": -0.10781855694949627, |
| "reward_after_std": 0.5886904541403055, |
| "reward_before_mean": 0.19626505579799414, |
| "reward_before_std": 0.5641138087958097, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3040836229920387, |
| "reward_change_min": -0.5314439944922924, |
| "reward_change_std": 0.2001748401671648, |
| "reward_std": 0.5886904690414667, |
| "rewards/accuracy_reward": 0.22916667349636555, |
| "rewards/cosine_scaled_reward": -0.03290162514895201, |
| "step": 47 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2589.458396911621, |
| "epoch": 0.054857142857142854, |
| "grad_norm": 0.07933427393436432, |
| "kl": 4.4988468289375305e-05, |
| "lambda_div_used": 0.6262567341327667, |
| "learning_rate": 9.509529358847654e-07, |
| "loss": -0.0358, |
| "reward": -0.13860276085324585, |
| "reward_after_mean": -0.13860276085324585, |
| "reward_after_std": 0.5960155855864286, |
| "reward_before_mean": 0.13786590658128262, |
| "reward_before_std": 0.5872014760971069, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2764686793088913, |
| "reward_change_min": -0.5099791921675205, |
| "reward_change_std": 0.19576375279575586, |
| "reward_std": 0.5960155855864286, |
| "rewards/accuracy_reward": 0.20833333767950535, |
| "rewards/cosine_scaled_reward": -0.07046743389219046, |
| "step": 48 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1985.2291946411133, |
| "epoch": 0.056, |
| "grad_norm": 0.12161526083946228, |
| "kl": 4.811584949493408e-05, |
| "lambda_div_used": 0.5859075263142586, |
| "learning_rate": 9.473264167865171e-07, |
| "loss": 0.0065, |
| "reward": -0.21219712868332863, |
| "reward_after_mean": -0.21219712868332863, |
| "reward_after_std": 0.439599821344018, |
| "reward_before_mean": 0.10536342021077871, |
| "reward_before_std": 0.39430073963012546, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3175605833530426, |
| "reward_change_min": -0.4657192714512348, |
| "reward_change_std": 0.19229275174438953, |
| "reward_std": 0.4395998399704695, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": -0.10296990955248475, |
| "step": 49 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2880.250011444092, |
| "epoch": 0.05714285714285714, |
| "grad_norm": 0.07723147422075272, |
| "kl": 3.2588839530944824e-05, |
| "lambda_div_used": 0.5741998106241226, |
| "learning_rate": 9.43578868212728e-07, |
| "loss": 0.0182, |
| "reward": -0.0913238637149334, |
| "reward_after_mean": -0.0913238637149334, |
| "reward_after_std": 0.42474013939499855, |
| "reward_before_mean": 0.3248649761080742, |
| "reward_before_std": 0.3397149038501084, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.41618884168565273, |
| "reward_change_min": -0.6034708395600319, |
| "reward_change_std": 0.23950859624892473, |
| "reward_std": 0.4247401561588049, |
| "rewards/accuracy_reward": 0.3125000074505806, |
| "rewards/cosine_scaled_reward": 0.012364983558654785, |
| "step": 50 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2433.020854949951, |
| "epoch": 0.05828571428571429, |
| "grad_norm": 0.09581972658634186, |
| "kl": 5.4717063903808594e-05, |
| "lambda_div_used": 0.5894461125135422, |
| "learning_rate": 9.397114317029974e-07, |
| "loss": -0.0308, |
| "reward": -0.28266510320827365, |
| "reward_after_mean": -0.28266510320827365, |
| "reward_after_std": 0.44258156418800354, |
| "reward_before_mean": 0.012647990137338638, |
| "reward_before_std": 0.40818152111023664, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29531308077275753, |
| "reward_change_min": -0.4761137217283249, |
| "reward_change_std": 0.1856073010712862, |
| "reward_std": 0.4425815735012293, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/cosine_scaled_reward": -0.15401867777109146, |
| "step": 51 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2707.250030517578, |
| "epoch": 0.05942857142857143, |
| "grad_norm": 0.14702580869197845, |
| "kl": 4.030205309391022e-05, |
| "lambda_div_used": 0.6197419166564941, |
| "learning_rate": 9.357252853159505e-07, |
| "loss": 0.0298, |
| "reward": 0.17821598052978516, |
| "reward_after_mean": 0.17821598052978516, |
| "reward_after_std": 0.6063419748097658, |
| "reward_before_mean": 0.629100788384676, |
| "reward_before_std": 0.5578461596742272, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4508847985416651, |
| "reward_change_min": -0.6935725994408131, |
| "reward_change_std": 0.2830996550619602, |
| "reward_std": 0.6063419822603464, |
| "rewards/accuracy_reward": 0.4583333432674408, |
| "rewards/cosine_scaled_reward": 0.17076744884252548, |
| "step": 52 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2799.8125610351562, |
| "epoch": 0.060571428571428575, |
| "grad_norm": 0.07566332817077637, |
| "kl": 5.0321221351623535e-05, |
| "lambda_div_used": 0.6389462202787399, |
| "learning_rate": 9.316216432703916e-07, |
| "loss": 0.0088, |
| "reward": -0.0043678306974470615, |
| "reward_after_mean": -0.0043678306974470615, |
| "reward_after_std": 0.6600480955094099, |
| "reward_before_mean": 0.31353550031781197, |
| "reward_before_std": 0.6461421558633447, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3179033286869526, |
| "reward_change_min": -0.5801984183490276, |
| "reward_change_std": 0.22101250104606152, |
| "reward_std": 0.6600481104105711, |
| "rewards/accuracy_reward": 0.29166667722165585, |
| "rewards/cosine_scaled_reward": 0.0218688128516078, |
| "step": 53 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2297.9583778381348, |
| "epoch": 0.061714285714285715, |
| "grad_norm": 0.09625467658042908, |
| "kl": 4.2378902435302734e-05, |
| "lambda_div_used": 0.6194662973284721, |
| "learning_rate": 9.274017555754407e-07, |
| "loss": 0.0224, |
| "reward": 0.0802488662302494, |
| "reward_after_mean": 0.0802488662302494, |
| "reward_after_std": 0.6323203574866056, |
| "reward_before_mean": 0.4949483387172222, |
| "reward_before_std": 0.5504263024777174, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.41469944082200527, |
| "reward_change_min": -0.6165112145245075, |
| "reward_change_std": 0.2416615542024374, |
| "reward_std": 0.632320374250412, |
| "rewards/accuracy_reward": 0.3541666753590107, |
| "rewards/cosine_scaled_reward": 0.1407816605642438, |
| "step": 54 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2896.416702270508, |
| "epoch": 0.06285714285714286, |
| "grad_norm": 0.05907125398516655, |
| "kl": 3.5256147384643555e-05, |
| "lambda_div_used": 0.602072462439537, |
| "learning_rate": 9.230669076497687e-07, |
| "loss": 0.0467, |
| "reward": -0.13429339788854122, |
| "reward_after_mean": -0.13429339788854122, |
| "reward_after_std": 0.57946902140975, |
| "reward_before_mean": 0.22700263070873916, |
| "reward_before_std": 0.4767341245897114, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36129603534936905, |
| "reward_change_min": -0.5782174952328205, |
| "reward_change_std": 0.21661545429378748, |
| "reward_std": 0.5794690307229757, |
| "rewards/accuracy_reward": 0.25000000186264515, |
| "rewards/cosine_scaled_reward": -0.02299738209694624, |
| "step": 55 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3075.187545776367, |
| "epoch": 0.064, |
| "grad_norm": 0.06310205906629562, |
| "kl": 4.1544437408447266e-05, |
| "lambda_div_used": 0.5549901723861694, |
| "learning_rate": 9.186184199300463e-07, |
| "loss": -0.0108, |
| "reward": -0.4740895018912852, |
| "reward_after_mean": -0.4740895018912852, |
| "reward_after_std": 0.32212498411536217, |
| "reward_before_mean": -0.2245362438261509, |
| "reward_before_std": 0.24502124171704054, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24955323711037636, |
| "reward_change_min": -0.3650141842663288, |
| "reward_change_std": 0.13405024446547031, |
| "reward_std": 0.3221249897032976, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.24536957405507565, |
| "step": 56 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3013.8541870117188, |
| "epoch": 0.06514285714285714, |
| "grad_norm": 0.052018001675605774, |
| "kl": 3.096461296081543e-05, |
| "lambda_div_used": 0.5674270242452621, |
| "learning_rate": 9.140576474687263e-07, |
| "loss": 0.0192, |
| "reward": -0.45193217881023884, |
| "reward_after_mean": -0.45193217881023884, |
| "reward_after_std": 0.37017360515892506, |
| "reward_before_mean": -0.21255622059106827, |
| "reward_before_std": 0.3090948835015297, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.23937593773007393, |
| "reward_change_min": -0.42212119325995445, |
| "reward_change_std": 0.14615233521908522, |
| "reward_std": 0.3701736144721508, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2542228833772242, |
| "step": 57 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2257.8750534057617, |
| "epoch": 0.06628571428571428, |
| "grad_norm": 0.09038142114877701, |
| "kl": 3.3486634492874146e-05, |
| "lambda_div_used": 0.6153116375207901, |
| "learning_rate": 9.093859795212817e-07, |
| "loss": 0.049, |
| "reward": -0.07369695231318474, |
| "reward_after_mean": -0.07369695231318474, |
| "reward_after_std": 0.5549236796796322, |
| "reward_before_mean": 0.2532845102250576, |
| "reward_before_std": 0.5286985114216805, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3269814867526293, |
| "reward_change_min": -0.537376407533884, |
| "reward_change_std": 0.20782853197306395, |
| "reward_std": 0.5549236983060837, |
| "rewards/accuracy_reward": 0.27083334513008595, |
| "rewards/cosine_scaled_reward": -0.017548808827996254, |
| "step": 58 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2807.6250381469727, |
| "epoch": 0.06742857142857143, |
| "grad_norm": 0.06863339245319366, |
| "kl": 3.577768802642822e-05, |
| "lambda_div_used": 0.6041718497872353, |
| "learning_rate": 9.046048391230247e-07, |
| "loss": -0.005, |
| "reward": -0.25026911310851574, |
| "reward_after_mean": -0.25026911310851574, |
| "reward_after_std": 0.5205795764923096, |
| "reward_before_mean": 0.026796480640769005, |
| "reward_before_std": 0.4837551396340132, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27706561237573624, |
| "reward_change_min": -0.5239802338182926, |
| "reward_change_std": 0.18853904772549868, |
| "reward_std": 0.5205795876681805, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.11903684702701867, |
| "step": 59 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2797.5833740234375, |
| "epoch": 0.06857142857142857, |
| "grad_norm": 0.07598927617073059, |
| "kl": 3.738701343536377e-05, |
| "lambda_div_used": 0.6269034072756767, |
| "learning_rate": 8.997156826556369e-07, |
| "loss": 0.0313, |
| "reward": -0.2369612492620945, |
| "reward_after_mean": -0.2369612492620945, |
| "reward_after_std": 0.6222166065126657, |
| "reward_before_mean": 0.005125788040459156, |
| "reward_before_std": 0.5912308068946004, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24208704754710197, |
| "reward_change_min": -0.48471353203058243, |
| "reward_change_std": 0.17278119549155235, |
| "reward_std": 0.6222166288644075, |
| "rewards/accuracy_reward": 0.14583333767950535, |
| "rewards/cosine_scaled_reward": -0.14070754405111074, |
| "step": 60 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2917.812515258789, |
| "epoch": 0.06971428571428571, |
| "grad_norm": 0.06223488971590996, |
| "kl": 3.0603259801864624e-05, |
| "lambda_div_used": 0.6177623867988586, |
| "learning_rate": 8.9471999940354e-07, |
| "loss": 0.0259, |
| "reward": -0.1357121616601944, |
| "reward_after_mean": -0.1357121616601944, |
| "reward_after_std": 0.5694319568574429, |
| "reward_before_mean": 0.16569078899919987, |
| "reward_before_std": 0.5441586868837476, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3014029450714588, |
| "reward_change_min": -0.5209132842719555, |
| "reward_change_std": 0.20053375512361526, |
| "reward_std": 0.5694319736212492, |
| "rewards/accuracy_reward": 0.20833333767950535, |
| "rewards/cosine_scaled_reward": -0.042642548214644194, |
| "step": 61 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2507.8958854675293, |
| "epoch": 0.07085714285714285, |
| "grad_norm": 0.0747612789273262, |
| "kl": 2.6777386665344238e-05, |
| "lambda_div_used": 0.61732517182827, |
| "learning_rate": 8.896193111002475e-07, |
| "loss": 0.0366, |
| "reward": 0.010478481650352478, |
| "reward_after_mean": 0.010478481650352478, |
| "reward_after_std": 0.6334415413439274, |
| "reward_before_mean": 0.4018659461289644, |
| "reward_before_std": 0.5490978918969631, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.39138743840157986, |
| "reward_change_min": -0.6315983533859253, |
| "reward_change_std": 0.24404066987335682, |
| "reward_std": 0.6334415581077337, |
| "rewards/accuracy_reward": 0.33333333767950535, |
| "rewards/cosine_scaled_reward": 0.06853258889168501, |
| "step": 62 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1902.5000228881836, |
| "epoch": 0.072, |
| "grad_norm": 0.07997458428144455, |
| "kl": 2.5197863578796387e-05, |
| "lambda_div_used": 0.6046535074710846, |
| "learning_rate": 8.844151714648274e-07, |
| "loss": 0.059, |
| "reward": 0.0217137411236763, |
| "reward_after_mean": 0.0217137411236763, |
| "reward_after_std": 0.5547807831317186, |
| "reward_before_mean": 0.42593196779489517, |
| "reward_before_std": 0.49076249497011304, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4042182229459286, |
| "reward_change_min": -0.6314646527171135, |
| "reward_change_std": 0.25159123074263334, |
| "reward_std": 0.5547807849943638, |
| "rewards/accuracy_reward": 0.3541666716337204, |
| "rewards/cosine_scaled_reward": 0.07176528126001358, |
| "step": 63 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2790.979217529297, |
| "epoch": 0.07314285714285715, |
| "grad_norm": 0.07697053253650665, |
| "kl": 4.419684410095215e-05, |
| "lambda_div_used": 0.6203151121735573, |
| "learning_rate": 8.791091657286267e-07, |
| "loss": 0.061, |
| "reward": -0.1100537832826376, |
| "reward_after_mean": -0.1100537832826376, |
| "reward_after_std": 0.5783387050032616, |
| "reward_before_mean": 0.19534806534647942, |
| "reward_before_std": 0.5548522733151913, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3054018337279558, |
| "reward_change_min": -0.5362299680709839, |
| "reward_change_std": 0.2037976048886776, |
| "reward_std": 0.5783387236297131, |
| "rewards/accuracy_reward": 0.20833333767950535, |
| "rewards/cosine_scaled_reward": -0.012985273322556168, |
| "step": 64 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2605.2708587646484, |
| "epoch": 0.07428571428571429, |
| "grad_norm": 0.0772913321852684, |
| "kl": 3.089010715484619e-05, |
| "lambda_div_used": 0.5563548430800438, |
| "learning_rate": 8.737029101523929e-07, |
| "loss": -0.0861, |
| "reward": -0.33125742711126804, |
| "reward_after_mean": -0.33125742711126804, |
| "reward_after_std": 0.3626741226762533, |
| "reward_before_mean": 0.005530592054128647, |
| "reward_before_std": 0.2566492212936282, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.33678802102804184, |
| "reward_change_min": -0.5241431556642056, |
| "reward_change_std": 0.18918895348906517, |
| "reward_std": 0.36267412453889847, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.161136067006737, |
| "step": 65 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2088.562511444092, |
| "epoch": 0.07542857142857143, |
| "grad_norm": 0.10951534658670425, |
| "kl": 2.690870314836502e-05, |
| "lambda_div_used": 0.5969364494085312, |
| "learning_rate": 8.681980515339463e-07, |
| "loss": 0.0287, |
| "reward": 0.012385480105876923, |
| "reward_after_mean": 0.012385480105876923, |
| "reward_after_std": 0.5128760654479265, |
| "reward_before_mean": 0.44090043008327484, |
| "reward_before_std": 0.4437296399846673, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4285149369388819, |
| "reward_change_min": -0.660201895982027, |
| "reward_change_std": 0.25701938942074776, |
| "reward_std": 0.512876084074378, |
| "rewards/accuracy_reward": 0.3750000111758709, |
| "rewards/cosine_scaled_reward": 0.0659004095941782, |
| "step": 66 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3402.8958740234375, |
| "epoch": 0.07657142857142857, |
| "grad_norm": 0.047973256558179855, |
| "kl": 3.771483898162842e-05, |
| "lambda_div_used": 0.5712955147027969, |
| "learning_rate": 8.625962667065487e-07, |
| "loss": -0.0027, |
| "reward": -0.4502652711234987, |
| "reward_after_mean": -0.4502652711234987, |
| "reward_after_std": 0.39950996078550816, |
| "reward_before_mean": -0.22067994717508554, |
| "reward_before_std": 0.3211147477850318, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.229585325345397, |
| "reward_change_min": -0.3470626436173916, |
| "reward_change_std": 0.12443333957344294, |
| "reward_std": 0.39950997941195965, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.26234661415219307, |
| "step": 67 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1642.9166984558105, |
| "epoch": 0.07771428571428571, |
| "grad_norm": 0.11412619799375534, |
| "kl": 2.551823854446411e-05, |
| "lambda_div_used": 0.6189040914177895, |
| "learning_rate": 8.568992620281243e-07, |
| "loss": -0.0912, |
| "reward": -0.22925877509987913, |
| "reward_after_mean": -0.22925877509987913, |
| "reward_after_std": 0.6055262424051762, |
| "reward_before_mean": 0.03262739907950163, |
| "reward_before_std": 0.5506744375452399, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2618861813098192, |
| "reward_change_min": -0.4676270857453346, |
| "reward_change_std": 0.16872322466224432, |
| "reward_std": 0.6055262610316277, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.09237260185182095, |
| "step": 68 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2018.6666946411133, |
| "epoch": 0.07885714285714286, |
| "grad_norm": 0.11094633489847183, |
| "kl": 4.1466206312179565e-05, |
| "lambda_div_used": 0.5734386518597603, |
| "learning_rate": 8.511087728614862e-07, |
| "loss": -0.0618, |
| "reward": -0.4316765144467354, |
| "reward_after_mean": -0.4316765144467354, |
| "reward_after_std": 0.4006939698010683, |
| "reward_before_mean": -0.1882182292174548, |
| "reward_before_std": 0.331131674349308, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24345828033983707, |
| "reward_change_min": -0.3763142116367817, |
| "reward_change_std": 0.13501812983304262, |
| "reward_std": 0.40069398283958435, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.22988490015268326, |
| "step": 69 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2860.8958587646484, |
| "epoch": 0.08, |
| "grad_norm": 0.06797178089618683, |
| "kl": 4.0102750062942505e-05, |
| "lambda_div_used": 0.6089852601289749, |
| "learning_rate": 8.452265630457282e-07, |
| "loss": 0.0416, |
| "reward": -0.16451344639062881, |
| "reward_after_mean": -0.16451344639062881, |
| "reward_after_std": 0.5175492316484451, |
| "reward_before_mean": 0.1350960824638605, |
| "reward_before_std": 0.5058744940906763, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2996095381677151, |
| "reward_change_min": -0.5449608005583286, |
| "reward_change_std": 0.2059446070343256, |
| "reward_std": 0.5175492409616709, |
| "rewards/accuracy_reward": 0.1875000037252903, |
| "rewards/cosine_scaled_reward": -0.05240389332175255, |
| "step": 70 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2700.2708587646484, |
| "epoch": 0.08114285714285714, |
| "grad_norm": 0.07971946895122528, |
| "kl": 4.7519803047180176e-05, |
| "lambda_div_used": 0.6114047914743423, |
| "learning_rate": 8.392544243589427e-07, |
| "loss": 0.0248, |
| "reward": -0.17357509583234787, |
| "reward_after_mean": -0.17357509583234787, |
| "reward_after_std": 0.5260436423122883, |
| "reward_before_mean": 0.1154884397983551, |
| "reward_before_std": 0.5224206217098981, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2890635374933481, |
| "reward_change_min": -0.5293963067233562, |
| "reward_change_std": 0.20269155222922564, |
| "reward_std": 0.5260436479002237, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.051178233698010445, |
| "step": 71 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2651.1250610351562, |
| "epoch": 0.08228571428571428, |
| "grad_norm": 0.1173771321773529, |
| "kl": 5.626678466796875e-05, |
| "lambda_div_used": 0.5625430718064308, |
| "learning_rate": 8.331941759724268e-07, |
| "loss": -0.0294, |
| "reward": -0.4364648088812828, |
| "reward_after_mean": -0.4364648088812828, |
| "reward_after_std": 0.3423657324165106, |
| "reward_before_mean": -0.17360325902700424, |
| "reward_before_std": 0.28492841869592667, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2628615368157625, |
| "reward_change_min": -0.44537778943777084, |
| "reward_change_std": 0.15708798076957464, |
| "reward_std": 0.3423657491803169, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2152699390426278, |
| "step": 72 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3339.9583740234375, |
| "epoch": 0.08342857142857144, |
| "grad_norm": 0.04790791496634483, |
| "kl": 4.526972770690918e-05, |
| "lambda_div_used": 0.6265770718455315, |
| "learning_rate": 8.270476638965461e-07, |
| "loss": 0.0626, |
| "reward": -0.12956268154084682, |
| "reward_after_mean": -0.12956268154084682, |
| "reward_after_std": 0.6143265012651682, |
| "reward_before_mean": 0.15906396182253957, |
| "reward_before_std": 0.5886987801641226, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.288626654073596, |
| "reward_change_min": -0.5101466961205006, |
| "reward_change_std": 0.1920691430568695, |
| "reward_std": 0.6143265273422003, |
| "rewards/accuracy_reward": 0.20833333767950535, |
| "rewards/cosine_scaled_reward": -0.049269367940723896, |
| "step": 73 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2385.7500381469727, |
| "epoch": 0.08457142857142858, |
| "grad_norm": 0.06600002944469452, |
| "kl": 3.542378544807434e-05, |
| "lambda_div_used": 0.6220900639891624, |
| "learning_rate": 8.208167604184217e-07, |
| "loss": -0.0312, |
| "reward": -0.0998319232603535, |
| "reward_after_mean": -0.0998319232603535, |
| "reward_after_std": 0.5906735248863697, |
| "reward_before_mean": 0.20307225361466408, |
| "reward_before_std": 0.566557977348566, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.30290416814386845, |
| "reward_change_min": -0.5291841961443424, |
| "reward_change_std": 0.20113296527415514, |
| "reward_std": 0.5906735453754663, |
| "rewards/accuracy_reward": 0.20833333767950535, |
| "rewards/cosine_scaled_reward": -0.005261080645141192, |
| "step": 74 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2732.458335876465, |
| "epoch": 0.08571428571428572, |
| "grad_norm": 0.059586018323898315, |
| "kl": 3.4804921597242355e-05, |
| "lambda_div_used": 0.5497717335820198, |
| "learning_rate": 8.145033635316128e-07, |
| "loss": -0.026, |
| "reward": -0.16143636964261532, |
| "reward_after_mean": -0.16143636964261532, |
| "reward_after_std": 0.41724423691630363, |
| "reward_before_mean": 0.31992355175316334, |
| "reward_before_std": 0.22394374571740627, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4813598971813917, |
| "reward_change_min": -0.6314556114375591, |
| "reward_change_std": 0.24685372970998287, |
| "reward_std": 0.4172442499548197, |
| "rewards/accuracy_reward": 0.27083333395421505, |
| "rewards/cosine_scaled_reward": 0.0490901879966259, |
| "step": 75 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2649.9791870117188, |
| "epoch": 0.08685714285714285, |
| "grad_norm": 0.07793173938989639, |
| "kl": 3.8176774978637695e-05, |
| "lambda_div_used": 0.5533522665500641, |
| "learning_rate": 8.081093963579707e-07, |
| "loss": 0.0329, |
| "reward": -0.4524771338328719, |
| "reward_after_mean": -0.4524771338328719, |
| "reward_after_std": 0.32186589390039444, |
| "reward_before_mean": -0.18054483737796545, |
| "reward_before_std": 0.23863591719418764, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2719323057681322, |
| "reward_change_min": -0.402019876986742, |
| "reward_change_std": 0.14489794615656137, |
| "reward_std": 0.32186589762568474, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.20137817226350307, |
| "step": 76 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2847.8333587646484, |
| "epoch": 0.088, |
| "grad_norm": 0.06544006615877151, |
| "kl": 4.89354133605957e-05, |
| "lambda_div_used": 0.5558431893587112, |
| "learning_rate": 8.01636806561836e-07, |
| "loss": -0.0184, |
| "reward": -0.46523131616413593, |
| "reward_after_mean": -0.46523131616413593, |
| "reward_after_std": 0.3304946720600128, |
| "reward_before_mean": -0.20099198445677757, |
| "reward_before_std": 0.2519041560590267, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26423933170735836, |
| "reward_change_min": -0.37857379391789436, |
| "reward_change_std": 0.140173084102571, |
| "reward_std": 0.3304946757853031, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.22182531282305717, |
| "step": 77 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3269.8333740234375, |
| "epoch": 0.08914285714285715, |
| "grad_norm": 0.05014213174581528, |
| "kl": 3.695487976074219e-05, |
| "lambda_div_used": 0.6150171384215355, |
| "learning_rate": 7.950875657567621e-07, |
| "loss": -0.0153, |
| "reward": -0.23963370453566313, |
| "reward_after_mean": -0.23963370453566313, |
| "reward_after_std": 0.5818531475961208, |
| "reward_before_mean": 0.01525909942574799, |
| "reward_before_std": 0.5286716222763062, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2548927925527096, |
| "reward_change_min": -0.411530327051878, |
| "reward_change_std": 0.1550313262268901, |
| "reward_std": 0.5818531513214111, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.10974090336821973, |
| "step": 78 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2262.062511444092, |
| "epoch": 0.09028571428571429, |
| "grad_norm": 0.06913956999778748, |
| "kl": 3.7103891372680664e-05, |
| "lambda_div_used": 0.5935603529214859, |
| "learning_rate": 7.884636689049422e-07, |
| "loss": 0.0312, |
| "reward": -0.23656679573468864, |
| "reward_after_mean": -0.23656679573468864, |
| "reward_after_std": 0.5612265523523092, |
| "reward_before_mean": 0.08522074297070503, |
| "reward_before_std": 0.4243400124832988, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3217875510454178, |
| "reward_change_min": -0.4353032112121582, |
| "reward_change_std": 0.16354462038725615, |
| "reward_std": 0.5612265765666962, |
| "rewards/accuracy_reward": 0.18750000186264515, |
| "rewards/cosine_scaled_reward": -0.10227925609797239, |
| "step": 79 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3292.3333587646484, |
| "epoch": 0.09142857142857143, |
| "grad_norm": 0.0538838729262352, |
| "kl": 4.382804036140442e-05, |
| "lambda_div_used": 0.5688970535993576, |
| "learning_rate": 7.817671337095244e-07, |
| "loss": 0.0288, |
| "reward": -0.3089722655713558, |
| "reward_after_mean": -0.3089722655713558, |
| "reward_after_std": 0.3733787778764963, |
| "reward_before_mean": 0.0013715587556362152, |
| "reward_before_std": 0.30914535373449326, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.31034383550286293, |
| "reward_change_min": -0.46327832341194153, |
| "reward_change_std": 0.17456937301903963, |
| "reward_std": 0.3733787890523672, |
| "rewards/accuracy_reward": 0.12500000558793545, |
| "rewards/cosine_scaled_reward": -0.12362844927702099, |
| "step": 80 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3043.270866394043, |
| "epoch": 0.09257142857142857, |
| "grad_norm": 0.08238786458969116, |
| "kl": 5.1021575927734375e-05, |
| "lambda_div_used": 0.5923707559704781, |
| "learning_rate": 7.75e-07, |
| "loss": 0.0152, |
| "reward": -0.26676796935498714, |
| "reward_after_mean": -0.26676796935498714, |
| "reward_after_std": 0.4838532619178295, |
| "reward_before_mean": 0.01490369625389576, |
| "reward_before_std": 0.4254928780719638, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28167167864739895, |
| "reward_change_min": -0.4777062237262726, |
| "reward_change_std": 0.17371418420225382, |
| "reward_std": 0.4838532619178295, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.11009630188345909, |
| "step": 81 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2630.625030517578, |
| "epoch": 0.09371428571428571, |
| "grad_norm": 0.06656210124492645, |
| "kl": 4.544854164123535e-05, |
| "lambda_div_used": 0.6220841184258461, |
| "learning_rate": 7.681643291108517e-07, |
| "loss": 0.0658, |
| "reward": -0.2009956305846572, |
| "reward_after_mean": -0.2009956305846572, |
| "reward_after_std": 0.6119217481464148, |
| "reward_before_mean": 0.05983926076442003, |
| "reward_before_std": 0.5658847019076347, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2608348857611418, |
| "reward_change_min": -0.45365019887685776, |
| "reward_change_std": 0.16699018515646458, |
| "reward_std": 0.6119217481464148, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.08599409856833518, |
| "step": 82 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2535.208381652832, |
| "epoch": 0.09485714285714286, |
| "grad_norm": 0.09338229149580002, |
| "kl": 5.167722702026367e-05, |
| "lambda_div_used": 0.5914888307452202, |
| "learning_rate": 7.612622032536507e-07, |
| "loss": 0.0644, |
| "reward": -0.14509151689708233, |
| "reward_after_mean": -0.14509151689708233, |
| "reward_after_std": 0.47357647120952606, |
| "reward_before_mean": 0.1998078590258956, |
| "reward_before_std": 0.4206119291484356, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3448993805795908, |
| "reward_change_min": -0.5211166813969612, |
| "reward_change_std": 0.20530468598008156, |
| "reward_std": 0.4735764730721712, |
| "rewards/accuracy_reward": 0.18750000558793545, |
| "rewards/cosine_scaled_reward": 0.012307855300605297, |
| "step": 83 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2915.8750076293945, |
| "epoch": 0.096, |
| "grad_norm": 0.07025005668401718, |
| "kl": 4.2825937271118164e-05, |
| "lambda_div_used": 0.6353590413928032, |
| "learning_rate": 7.54295724882796e-07, |
| "loss": -0.0466, |
| "reward": -0.1503741154447198, |
| "reward_after_mean": -0.1503741154447198, |
| "reward_after_std": 0.6686306204646826, |
| "reward_before_mean": 0.10233869170770049, |
| "reward_before_std": 0.6240420090034604, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25271278992295265, |
| "reward_change_min": -0.43359045311808586, |
| "reward_change_std": 0.1594225950539112, |
| "reward_std": 0.6686306446790695, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": -0.10599466459825635, |
| "step": 84 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2968.9375610351562, |
| "epoch": 0.09714285714285714, |
| "grad_norm": 0.05030859634280205, |
| "kl": 2.9653310775756836e-05, |
| "lambda_div_used": 0.6120947226881981, |
| "learning_rate": 7.472670160550848e-07, |
| "loss": 0.02, |
| "reward": -0.31006562570109963, |
| "reward_after_mean": -0.31006562570109963, |
| "reward_after_std": 0.5445140562951565, |
| "reward_before_mean": -0.0690727960318327, |
| "reward_before_std": 0.5210688021034002, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2409928422421217, |
| "reward_change_min": -0.46906592324376106, |
| "reward_change_std": 0.1683354014530778, |
| "reward_std": 0.5445140581578016, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.17323945881798863, |
| "step": 85 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2726.9167098999023, |
| "epoch": 0.09828571428571428, |
| "grad_norm": 0.08248579502105713, |
| "kl": 5.6862831115722656e-05, |
| "lambda_div_used": 0.5853299722075462, |
| "learning_rate": 7.401782177833147e-07, |
| "loss": -0.015, |
| "reward": -0.32714659720659256, |
| "reward_after_mean": -0.32714659720659256, |
| "reward_after_std": 0.4505470525473356, |
| "reward_before_mean": -0.05685961013659835, |
| "reward_before_std": 0.39618763769976795, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2702869772911072, |
| "reward_change_min": -0.49061961844563484, |
| "reward_change_std": 0.17353465128690004, |
| "reward_std": 0.4505470544099808, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.14019294315949082, |
| "step": 86 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2504.604232788086, |
| "epoch": 0.09942857142857142, |
| "grad_norm": 0.07946456968784332, |
| "kl": 5.137920379638672e-05, |
| "lambda_div_used": 0.5860441103577614, |
| "learning_rate": 7.330314893841101e-07, |
| "loss": -0.0658, |
| "reward": -0.25398197025060654, |
| "reward_after_mean": -0.25398197025060654, |
| "reward_after_std": 0.4263457953929901, |
| "reward_before_mean": 0.039198137819767, |
| "reward_before_std": 0.4029952948912978, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2931801360100508, |
| "reward_change_min": -0.46910280734300613, |
| "reward_change_std": 0.18813357036560774, |
| "reward_std": 0.4263457991182804, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.1066351905465126, |
| "step": 87 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2021.3958702087402, |
| "epoch": 0.10057142857142858, |
| "grad_norm": 0.08914105594158173, |
| "kl": 3.445148468017578e-05, |
| "lambda_div_used": 0.6275195479393005, |
| "learning_rate": 7.258290078201731e-07, |
| "loss": 0.0661, |
| "reward": -0.03901347843930125, |
| "reward_after_mean": -0.03901347843930125, |
| "reward_after_std": 0.6304188724607229, |
| "reward_before_mean": 0.28059265296906233, |
| "reward_before_std": 0.5838784109801054, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3196061383932829, |
| "reward_change_min": -0.4850176088511944, |
| "reward_change_std": 0.19270309899002314, |
| "reward_std": 0.6304188761860132, |
| "rewards/accuracy_reward": 0.25000000931322575, |
| "rewards/cosine_scaled_reward": 0.030592639930546284, |
| "step": 88 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2855.666702270508, |
| "epoch": 0.10171428571428572, |
| "grad_norm": 0.052225928753614426, |
| "kl": 3.810226917266846e-05, |
| "lambda_div_used": 0.603381521999836, |
| "learning_rate": 7.185729670371604e-07, |
| "loss": -0.0748, |
| "reward": -0.2931769546121359, |
| "reward_after_mean": -0.2931769546121359, |
| "reward_after_std": 0.5186727736145258, |
| "reward_before_mean": -0.04963091528043151, |
| "reward_before_std": 0.4795092437416315, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2435460388660431, |
| "reward_change_min": -0.4475689232349396, |
| "reward_change_std": 0.1609211042523384, |
| "reward_std": 0.5186727829277515, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.17463092133402824, |
| "step": 89 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2377.9375228881836, |
| "epoch": 0.10285714285714286, |
| "grad_norm": 0.1115579754114151, |
| "kl": 6.511807441711426e-05, |
| "lambda_div_used": 0.5806571692228317, |
| "learning_rate": 7.11265577295385e-07, |
| "loss": 0.1049, |
| "reward": -0.39063363266177475, |
| "reward_after_mean": -0.39063363266177475, |
| "reward_after_std": 0.42313366010785103, |
| "reward_before_mean": -0.14117828011512756, |
| "reward_before_std": 0.3631622865796089, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24945535697042942, |
| "reward_change_min": -0.38458868488669395, |
| "reward_change_std": 0.14130910206586123, |
| "reward_std": 0.4231336638331413, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.1828449461609125, |
| "step": 90 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2758.250015258789, |
| "epoch": 0.104, |
| "grad_norm": 0.0690622553229332, |
| "kl": 4.595518112182617e-05, |
| "lambda_div_used": 0.5834402665495872, |
| "learning_rate": 7.039090644965509e-07, |
| "loss": 0.0142, |
| "reward": -0.1535566644743085, |
| "reward_after_mean": -0.1535566644743085, |
| "reward_after_std": 0.4708296097815037, |
| "reward_before_mean": 0.22540673054754734, |
| "reward_before_std": 0.3833985608071089, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3789634220302105, |
| "reward_change_min": -0.6174517869949341, |
| "reward_change_std": 0.2261828240007162, |
| "reward_std": 0.4708296228200197, |
| "rewards/accuracy_reward": 0.2291666679084301, |
| "rewards/cosine_scaled_reward": -0.003759911749511957, |
| "step": 91 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2467.2083892822266, |
| "epoch": 0.10514285714285715, |
| "grad_norm": 0.07518215477466583, |
| "kl": 3.6522746086120605e-05, |
| "lambda_div_used": 0.5778974890708923, |
| "learning_rate": 6.965056695057204e-07, |
| "loss": 0.0134, |
| "reward": -0.324931837618351, |
| "reward_after_mean": -0.324931837618351, |
| "reward_after_std": 0.42975695990025997, |
| "reward_before_mean": -0.03748153988271952, |
| "reward_before_std": 0.35468481201678514, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2874503042548895, |
| "reward_change_min": -0.4382200054824352, |
| "reward_change_std": 0.1673103515058756, |
| "reward_std": 0.42975698225200176, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.14164821058511734, |
| "step": 92 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3584.0, |
| "epoch": 0.10628571428571429, |
| "grad_norm": 0.05293423309922218, |
| "kl": 5.128979682922363e-05, |
| "lambda_div_used": 0.5581478402018547, |
| "learning_rate": 6.890576474687263e-07, |
| "loss": -0.0, |
| "reward": -0.4109080731868744, |
| "reward_after_mean": -0.4109080731868744, |
| "reward_after_std": 0.33292335644364357, |
| "reward_before_mean": -0.12180843483656645, |
| "reward_before_std": 0.26414576172828674, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28909964114427567, |
| "reward_change_min": -0.45860791578888893, |
| "reward_change_std": 0.16918409056961536, |
| "reward_std": 0.332923362031579, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.16347510018385947, |
| "step": 93 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2337.0625076293945, |
| "epoch": 0.10742857142857143, |
| "grad_norm": 0.08120116591453552, |
| "kl": 4.356354475021362e-05, |
| "lambda_div_used": 0.5623572915792465, |
| "learning_rate": 6.815672671252315e-07, |
| "loss": 0.0672, |
| "reward": -0.2148991823196411, |
| "reward_after_mean": -0.2148991823196411, |
| "reward_after_std": 0.37764647975564003, |
| "reward_before_mean": 0.17423074319958687, |
| "reward_before_std": 0.2815079055726528, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3891299031674862, |
| "reward_change_min": -0.5794133953750134, |
| "reward_change_std": 0.21886237617582083, |
| "reward_std": 0.3776464983820915, |
| "rewards/accuracy_reward": 0.2083333358168602, |
| "rewards/cosine_scaled_reward": -0.034102603793144226, |
| "step": 94 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3268.7708587646484, |
| "epoch": 0.10857142857142857, |
| "grad_norm": 0.048412173986434937, |
| "kl": 3.3482909202575684e-05, |
| "lambda_div_used": 0.599761851131916, |
| "learning_rate": 6.740368101176495e-07, |
| "loss": 0.0138, |
| "reward": -0.28226344753056765, |
| "reward_after_mean": -0.28226344753056765, |
| "reward_after_std": 0.5015552807599306, |
| "reward_before_mean": -0.01817359635606408, |
| "reward_before_std": 0.4594444427639246, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2640898581594229, |
| "reward_change_min": -0.45775213465094566, |
| "reward_change_std": 0.170160255394876, |
| "reward_std": 0.5015552863478661, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.14317359682172537, |
| "step": 95 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2311.229202270508, |
| "epoch": 0.10971428571428571, |
| "grad_norm": 0.08286502212285995, |
| "kl": 3.9830803871154785e-05, |
| "lambda_div_used": 0.6152824014425278, |
| "learning_rate": 6.664685702961344e-07, |
| "loss": 0.0394, |
| "reward": -0.1798182651400566, |
| "reward_after_mean": -0.1798182651400566, |
| "reward_after_std": 0.5684184953570366, |
| "reward_before_mean": 0.09630595671478659, |
| "reward_before_std": 0.5319525888189673, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2761242166161537, |
| "reward_change_min": -0.4500977620482445, |
| "reward_change_std": 0.17272682767361403, |
| "reward_std": 0.5684185232967138, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/cosine_scaled_reward": -0.11202737595885992, |
| "step": 96 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3088.104202270508, |
| "epoch": 0.11085714285714286, |
| "grad_norm": 0.05773423984646797, |
| "kl": 4.525482654571533e-05, |
| "lambda_div_used": 0.5830484703183174, |
| "learning_rate": 6.588648530198504e-07, |
| "loss": 0.012, |
| "reward": -0.24631217122077942, |
| "reward_after_mean": -0.24631217122077942, |
| "reward_after_std": 0.4177880808711052, |
| "reward_before_mean": 0.052551381289958954, |
| "reward_before_std": 0.3777043428272009, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2988635450601578, |
| "reward_change_min": -0.4822593107819557, |
| "reward_change_std": 0.18344944156706333, |
| "reward_std": 0.4177880957722664, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.11411529779434204, |
| "step": 97 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3099.6666870117188, |
| "epoch": 0.112, |
| "grad_norm": 0.05776653811335564, |
| "kl": 4.242360591888428e-05, |
| "lambda_div_used": 0.5548205152153969, |
| "learning_rate": 6.512279744547392e-07, |
| "loss": 0.0853, |
| "reward": -0.30739316577091813, |
| "reward_after_mean": -0.30739316577091813, |
| "reward_after_std": 0.3838734310120344, |
| "reward_before_mean": 0.044590696692466736, |
| "reward_before_std": 0.2453754236921668, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3519838694483042, |
| "reward_change_min": -0.49566248431801796, |
| "reward_change_std": 0.18073826655745506, |
| "reward_std": 0.38387343287467957, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.10124263912439346, |
| "step": 98 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2814.0416679382324, |
| "epoch": 0.11314285714285714, |
| "grad_norm": 0.10099756717681885, |
| "kl": 3.9711594581604004e-05, |
| "lambda_div_used": 0.589689776301384, |
| "learning_rate": 6.435602608679916e-07, |
| "loss": -0.0116, |
| "reward": -0.24916072934865952, |
| "reward_after_mean": -0.24916072934865952, |
| "reward_after_std": 0.5363867282867432, |
| "reward_before_mean": 0.07555947656510398, |
| "reward_before_std": 0.4080575914122164, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.32472023367881775, |
| "reward_change_min": -0.46621018648147583, |
| "reward_change_std": 0.16967851482331753, |
| "reward_std": 0.5363867320120335, |
| "rewards/accuracy_reward": 0.18750000186264515, |
| "rewards/cosine_scaled_reward": -0.11194050963968039, |
| "step": 99 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2585.208366394043, |
| "epoch": 0.11428571428571428, |
| "grad_norm": 0.08013861626386642, |
| "kl": 3.557652235031128e-05, |
| "lambda_div_used": 0.6440299674868584, |
| "learning_rate": 6.358640479194451e-07, |
| "loss": 0.0233, |
| "reward": -0.017059004865586758, |
| "reward_after_mean": -0.017059004865586758, |
| "reward_after_std": 0.6788044832646847, |
| "reward_before_mean": 0.27609538938850164, |
| "reward_before_std": 0.672226045280695, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29315441474318504, |
| "reward_change_min": -0.560445386916399, |
| "reward_change_std": 0.2121435971930623, |
| "reward_std": 0.6788045018911362, |
| "rewards/accuracy_reward": 0.25000000186264515, |
| "rewards/cosine_scaled_reward": 0.02609538659453392, |
| "step": 100 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2698.687515258789, |
| "epoch": 0.11542857142857142, |
| "grad_norm": 0.08196399360895157, |
| "kl": 4.9740076065063477e-05, |
| "lambda_div_used": 0.5603137612342834, |
| "learning_rate": 6.281416799501187e-07, |
| "loss": -0.0552, |
| "reward": -0.2113272361457348, |
| "reward_after_mean": -0.2113272361457348, |
| "reward_after_std": 0.3802106771618128, |
| "reward_before_mean": 0.176470085978508, |
| "reward_before_std": 0.27200379874557257, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3877973351627588, |
| "reward_change_min": -0.5599614717066288, |
| "reward_change_std": 0.21361587569117546, |
| "reward_std": 0.38021068647503853, |
| "rewards/accuracy_reward": 0.2083333358168602, |
| "rewards/cosine_scaled_reward": -0.03186323493719101, |
| "step": 101 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2008.1875457763672, |
| "epoch": 0.11657142857142858, |
| "grad_norm": 0.09427309036254883, |
| "kl": 3.542006015777588e-05, |
| "lambda_div_used": 0.5826256647706032, |
| "learning_rate": 6.203955092681039e-07, |
| "loss": -0.0045, |
| "reward": -0.2004177300259471, |
| "reward_after_mean": -0.2004177300259471, |
| "reward_after_std": 0.498240664601326, |
| "reward_before_mean": 0.16466995794326067, |
| "reward_before_std": 0.37351171765476465, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36508770659565926, |
| "reward_change_min": -0.5195483043789864, |
| "reward_change_std": 0.19531819131225348, |
| "reward_std": 0.4982406720519066, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.001996707171201706, |
| "step": 102 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2895.250020980835, |
| "epoch": 0.11771428571428572, |
| "grad_norm": 0.06802447140216827, |
| "kl": 4.501640796661377e-05, |
| "lambda_div_used": 0.5506640374660492, |
| "learning_rate": 6.126278954320294e-07, |
| "loss": 0.0391, |
| "reward": -0.3214366286993027, |
| "reward_after_mean": -0.3214366286993027, |
| "reward_after_std": 0.3532369527965784, |
| "reward_before_mean": 0.050119780004024506, |
| "reward_before_std": 0.22584082814864814, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3715564049780369, |
| "reward_change_min": -0.520975548774004, |
| "reward_change_std": 0.19540261384099722, |
| "reward_std": 0.35323695838451385, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.09571357164531946, |
| "step": 103 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2521.3333435058594, |
| "epoch": 0.11885714285714286, |
| "grad_norm": 0.096355140209198, |
| "kl": 4.947185516357422e-05, |
| "lambda_div_used": 0.5742901861667633, |
| "learning_rate": 6.048412045323164e-07, |
| "loss": 0.0001, |
| "reward": -0.32155255414545536, |
| "reward_after_mean": -0.32155255414545536, |
| "reward_after_std": 0.39902236871421337, |
| "reward_before_mean": -0.03546402044594288, |
| "reward_before_std": 0.3418318200856447, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2860885336995125, |
| "reward_change_min": -0.4318212755024433, |
| "reward_change_std": 0.16667354479432106, |
| "reward_std": 0.39902237243950367, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.13963068462908268, |
| "step": 104 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2526.000030517578, |
| "epoch": 0.12, |
| "grad_norm": 0.07842111587524414, |
| "kl": 4.8801302909851074e-05, |
| "lambda_div_used": 0.6022165417671204, |
| "learning_rate": 5.97037808470444e-07, |
| "loss": 0.0569, |
| "reward": -0.12751667387783527, |
| "reward_after_mean": -0.12751667387783527, |
| "reward_after_std": 0.582199590280652, |
| "reward_before_mean": 0.24656505044549704, |
| "reward_before_std": 0.4756303254980594, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.37408171594142914, |
| "reward_change_min": -0.5777854695916176, |
| "reward_change_std": 0.22329542227089405, |
| "reward_std": 0.5821996051818132, |
| "rewards/accuracy_reward": 0.25000000186264515, |
| "rewards/cosine_scaled_reward": -0.0034349607303738594, |
| "step": 105 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2396.291702270508, |
| "epoch": 0.12114285714285715, |
| "grad_norm": 0.07387516647577286, |
| "kl": 2.6203226298093796e-05, |
| "lambda_div_used": 0.6128493994474411, |
| "learning_rate": 5.892200842364462e-07, |
| "loss": -0.0293, |
| "reward": 0.159407502040267, |
| "reward_after_mean": 0.159407502040267, |
| "reward_after_std": 0.5759131647646427, |
| "reward_before_mean": 0.6274341251701117, |
| "reward_before_std": 0.5208892030641437, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.46802657656371593, |
| "reward_change_min": -0.7112628631293774, |
| "reward_change_std": 0.2871347274631262, |
| "reward_std": 0.5759131908416748, |
| "rewards/accuracy_reward": 0.4583333469927311, |
| "rewards/cosine_scaled_reward": 0.16910075349733233, |
| "step": 106 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2806.875045776367, |
| "epoch": 0.12228571428571429, |
| "grad_norm": 0.08797896653413773, |
| "kl": 4.5686960220336914e-05, |
| "lambda_div_used": 0.5551519840955734, |
| "learning_rate": 5.813904131848564e-07, |
| "loss": -0.0206, |
| "reward": -0.20906101167201996, |
| "reward_after_mean": -0.20906101167201996, |
| "reward_after_std": 0.35940215550363064, |
| "reward_before_mean": 0.19015613943338394, |
| "reward_before_std": 0.24773720651865005, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.39921717159450054, |
| "reward_change_min": -0.5645803362131119, |
| "reward_change_std": 0.21728475205600262, |
| "reward_std": 0.35940215922892094, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/cosine_scaled_reward": -0.03901051543653011, |
| "step": 107 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2676.3125381469727, |
| "epoch": 0.12342857142857143, |
| "grad_norm": 0.08117339015007019, |
| "kl": 4.683062434196472e-05, |
| "lambda_div_used": 0.5782058760523796, |
| "learning_rate": 5.735511803093248e-07, |
| "loss": 0.0222, |
| "reward": -0.33698799335979857, |
| "reward_after_mean": -0.33698799335979857, |
| "reward_after_std": 0.4150846730917692, |
| "reward_before_mean": -0.06898907572031021, |
| "reward_before_std": 0.3580573983490467, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2679989282041788, |
| "reward_change_min": -0.39731478318572044, |
| "reward_change_std": 0.15669311955571175, |
| "reward_std": 0.41508468240499496, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.17315573990345, |
| "step": 108 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3091.4166870117188, |
| "epoch": 0.12457142857142857, |
| "grad_norm": 0.06771776080131531, |
| "kl": 4.081428050994873e-05, |
| "lambda_div_used": 0.5580320879817009, |
| "learning_rate": 5.657047735161255e-07, |
| "loss": -0.0162, |
| "reward": -0.30369802191853523, |
| "reward_after_mean": -0.30369802191853523, |
| "reward_after_std": 0.3783009462058544, |
| "reward_before_mean": 0.060508192516863346, |
| "reward_before_std": 0.2597539462149143, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36420623771846294, |
| "reward_change_min": -0.5151935815811157, |
| "reward_change_std": 0.19158014561980963, |
| "reward_std": 0.37830095551908016, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.08532514423131943, |
| "step": 109 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2727.875045776367, |
| "epoch": 0.12571428571428572, |
| "grad_norm": 0.07848876714706421, |
| "kl": 3.8996338844299316e-05, |
| "lambda_div_used": 0.5898331105709076, |
| "learning_rate": 5.578535828967777e-07, |
| "loss": 0.0059, |
| "reward": -0.27789773186668754, |
| "reward_after_mean": -0.27789773186668754, |
| "reward_after_std": 0.48964778520166874, |
| "reward_before_mean": 0.006700331810861826, |
| "reward_before_std": 0.40999170672148466, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28459805622696877, |
| "reward_change_min": -0.4541938379406929, |
| "reward_change_std": 0.16897330060601234, |
| "reward_std": 0.48964780382812023, |
| "rewards/accuracy_reward": 0.16666667349636555, |
| "rewards/cosine_scaled_reward": -0.15996634401381016, |
| "step": 110 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3152.3958587646484, |
| "epoch": 0.12685714285714286, |
| "grad_norm": 0.06026454642415047, |
| "kl": 5.206465721130371e-05, |
| "lambda_div_used": 0.6035187616944313, |
| "learning_rate": 5.5e-07, |
| "loss": 0.0004, |
| "reward": -0.22968922927975655, |
| "reward_after_mean": -0.22968922927975655, |
| "reward_after_std": 0.5089491438120604, |
| "reward_before_mean": 0.050578076392412186, |
| "reward_before_std": 0.47721442952752113, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28026728704571724, |
| "reward_change_min": -0.4855511710047722, |
| "reward_change_std": 0.1824399819597602, |
| "reward_std": 0.5089491568505764, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.09525526221841574, |
| "step": 111 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3253.937530517578, |
| "epoch": 0.128, |
| "grad_norm": 0.054045893251895905, |
| "kl": 4.832446575164795e-05, |
| "lambda_div_used": 0.6144762486219406, |
| "learning_rate": 5.421464171032224e-07, |
| "loss": 0.0014, |
| "reward": -0.07707784557715058, |
| "reward_after_mean": -0.07707784557715058, |
| "reward_after_std": 0.5871799997985363, |
| "reward_before_mean": 0.2822006791830063, |
| "reward_before_std": 0.5333117246627808, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.35927851870656013, |
| "reward_change_min": -0.6481843888759613, |
| "reward_change_std": 0.23641589283943176, |
| "reward_std": 0.5871800072491169, |
| "rewards/accuracy_reward": 0.2708333358168602, |
| "rewards/cosine_scaled_reward": 0.01136732567101717, |
| "step": 112 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2541.812545776367, |
| "epoch": 0.12914285714285714, |
| "grad_norm": 0.09499017894268036, |
| "kl": 4.720315337181091e-05, |
| "lambda_div_used": 0.5564066097140312, |
| "learning_rate": 5.342952264838747e-07, |
| "loss": 0.0471, |
| "reward": -0.3626333475112915, |
| "reward_after_mean": -0.3626333475112915, |
| "reward_after_std": 0.3075137473642826, |
| "reward_before_mean": -0.05370184499770403, |
| "reward_before_std": 0.25310691073536873, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.30893150344491005, |
| "reward_change_min": -0.4691920429468155, |
| "reward_change_std": 0.1770205283537507, |
| "reward_std": 0.3075137585401535, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.13703518453985453, |
| "step": 113 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2547.5416984558105, |
| "epoch": 0.13028571428571428, |
| "grad_norm": 0.06982935965061188, |
| "kl": 2.650544047355652e-05, |
| "lambda_div_used": 0.5948992818593979, |
| "learning_rate": 5.264488196906752e-07, |
| "loss": 0.0506, |
| "reward": -0.33033538423478603, |
| "reward_after_mean": -0.33033538423478603, |
| "reward_after_std": 0.49076898768544197, |
| "reward_before_mean": -0.08067071554251015, |
| "reward_before_std": 0.4387869122438133, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2496646661311388, |
| "reward_change_min": -0.38466524705290794, |
| "reward_change_std": 0.15147447120398283, |
| "reward_std": 0.4907689895480871, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.18483738787472248, |
| "step": 114 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2926.3125228881836, |
| "epoch": 0.13142857142857142, |
| "grad_norm": 0.10055308789014816, |
| "kl": 4.2185187339782715e-05, |
| "lambda_div_used": 0.5866656303405762, |
| "learning_rate": 5.186095868151436e-07, |
| "loss": 0.0219, |
| "reward": -0.16912975907325745, |
| "reward_after_mean": -0.16912975907325745, |
| "reward_after_std": 0.4218728318810463, |
| "reward_before_mean": 0.17230269685387611, |
| "reward_before_std": 0.39482294395565987, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34143244847655296, |
| "reward_change_min": -0.5322228632867336, |
| "reward_change_std": 0.21214590221643448, |
| "reward_std": 0.4218728430569172, |
| "rewards/accuracy_reward": 0.2708333432674408, |
| "rewards/cosine_scaled_reward": -0.09853065386414528, |
| "step": 115 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3185.895835876465, |
| "epoch": 0.13257142857142856, |
| "grad_norm": 0.06827546656131744, |
| "kl": 4.4345855712890625e-05, |
| "lambda_div_used": 0.579861506819725, |
| "learning_rate": 5.107799157635538e-07, |
| "loss": -0.017, |
| "reward": -0.36347829084843397, |
| "reward_after_mean": -0.36347829084843397, |
| "reward_after_std": 0.425207169726491, |
| "reward_before_mean": -0.10740169882774353, |
| "reward_before_std": 0.36591998394578695, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25607660599052906, |
| "reward_change_min": -0.4011174105107784, |
| "reward_change_std": 0.15214112866669893, |
| "reward_std": 0.4252071734517813, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.19073502300307155, |
| "step": 116 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3265.666702270508, |
| "epoch": 0.1337142857142857, |
| "grad_norm": 0.058839015662670135, |
| "kl": 5.224347114562988e-05, |
| "lambda_div_used": 0.5589891448616982, |
| "learning_rate": 5.02962191529556e-07, |
| "loss": 0.0144, |
| "reward": -0.47565145045518875, |
| "reward_after_mean": -0.47565145045518875, |
| "reward_after_std": 0.34672038443386555, |
| "reward_before_mean": -0.2341720014810562, |
| "reward_before_std": 0.2644944768399, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2414794433861971, |
| "reward_change_min": -0.3730311393737793, |
| "reward_change_std": 0.13014927878975868, |
| "reward_std": 0.3467204011976719, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2550053410232067, |
| "step": 117 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3087.9375610351562, |
| "epoch": 0.13485714285714287, |
| "grad_norm": 0.05542779713869095, |
| "kl": 3.71783971786499e-05, |
| "lambda_div_used": 0.6410617902874947, |
| "learning_rate": 4.951587954676837e-07, |
| "loss": 0.0602, |
| "reward": 0.02640039217658341, |
| "reward_after_mean": 0.02640039217658341, |
| "reward_after_std": 0.6670121420174837, |
| "reward_before_mean": 0.34903212962672114, |
| "reward_before_std": 0.6545964349061251, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.322631748393178, |
| "reward_change_min": -0.5322613082826138, |
| "reward_change_std": 0.21465477347373962, |
| "reward_std": 0.66701215878129, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": 0.07819879939779639, |
| "step": 118 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2036.8542251586914, |
| "epoch": 0.136, |
| "grad_norm": 0.09512177854776382, |
| "kl": 4.920363426208496e-05, |
| "lambda_div_used": 0.5765073597431183, |
| "learning_rate": 4.873721045679706e-07, |
| "loss": 0.0968, |
| "reward": -0.1327025555074215, |
| "reward_after_mean": -0.1327025555074215, |
| "reward_after_std": 0.4383635278791189, |
| "reward_before_mean": 0.2736157886683941, |
| "reward_before_std": 0.34712607227265835, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4063183292746544, |
| "reward_change_min": -0.5770336836576462, |
| "reward_change_std": 0.22668993193656206, |
| "reward_std": 0.43836353346705437, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": 0.0444490984082222, |
| "step": 119 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2693.6458854675293, |
| "epoch": 0.13714285714285715, |
| "grad_norm": 0.0801864042878151, |
| "kl": 5.1856040954589844e-05, |
| "lambda_div_used": 0.5954036563634872, |
| "learning_rate": 4.79604490731896e-07, |
| "loss": -0.0416, |
| "reward": -0.10674675926566124, |
| "reward_after_mean": -0.10674675926566124, |
| "reward_after_std": 0.5345237273722887, |
| "reward_before_mean": 0.26713848020881414, |
| "reward_before_std": 0.44027570402249694, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.37388524040579796, |
| "reward_change_min": -0.5998699106276035, |
| "reward_change_std": 0.22230207175016403, |
| "reward_std": 0.5345237515866756, |
| "rewards/accuracy_reward": 0.25000000186264515, |
| "rewards/cosine_scaled_reward": 0.017138468101620674, |
| "step": 120 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1960.8958587646484, |
| "epoch": 0.1382857142857143, |
| "grad_norm": 0.09297072887420654, |
| "kl": 3.6925775930285454e-05, |
| "lambda_div_used": 0.5758452340960503, |
| "learning_rate": 4.7185832004988133e-07, |
| "loss": -0.0275, |
| "reward": -0.15540813654661179, |
| "reward_after_mean": -0.15540813654661179, |
| "reward_after_std": 0.3896722886711359, |
| "reward_before_mean": 0.20818629674613476, |
| "reward_before_std": 0.34600438084453344, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3635944165289402, |
| "reward_change_min": -0.5350156500935555, |
| "reward_change_std": 0.21344942972064018, |
| "reward_std": 0.38967230543494225, |
| "rewards/accuracy_reward": 0.1875000074505806, |
| "rewards/cosine_scaled_reward": 0.020686281844973564, |
| "step": 121 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2996.7917289733887, |
| "epoch": 0.13942857142857143, |
| "grad_norm": 0.07053450495004654, |
| "kl": 5.537271499633789e-05, |
| "lambda_div_used": 0.604009747505188, |
| "learning_rate": 4.641359520805548e-07, |
| "loss": 0.0244, |
| "reward": -0.18336665583774447, |
| "reward_after_mean": -0.18336665583774447, |
| "reward_after_std": 0.5550395585596561, |
| "reward_before_mean": 0.11408489104360342, |
| "reward_before_std": 0.4755072835832834, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29745154455304146, |
| "reward_change_min": -0.4127044528722763, |
| "reward_change_std": 0.16168679296970367, |
| "reward_std": 0.5550395771861076, |
| "rewards/accuracy_reward": 0.16666667349636555, |
| "rewards/cosine_scaled_reward": -0.05258178571239114, |
| "step": 122 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2965.687530517578, |
| "epoch": 0.14057142857142857, |
| "grad_norm": 0.061138082295656204, |
| "kl": 4.2922794818878174e-05, |
| "lambda_div_used": 0.5864768177270889, |
| "learning_rate": 4.5643973913200837e-07, |
| "loss": 0.0629, |
| "reward": -0.24433407932519913, |
| "reward_after_mean": -0.24433407932519913, |
| "reward_after_std": 0.43522679433226585, |
| "reward_before_mean": 0.05285754054784775, |
| "reward_before_std": 0.4003843404352665, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2971916198730469, |
| "reward_change_min": -0.4781832844018936, |
| "reward_change_std": 0.18817945942282677, |
| "reward_std": 0.435226796194911, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.09297579154372215, |
| "step": 123 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2273.895881652832, |
| "epoch": 0.1417142857142857, |
| "grad_norm": 0.06792795658111572, |
| "kl": 2.843700349330902e-05, |
| "lambda_div_used": 0.5992666333913803, |
| "learning_rate": 4.4877202554526084e-07, |
| "loss": 0.0724, |
| "reward": -0.030117375776171684, |
| "reward_after_mean": -0.030117375776171684, |
| "reward_after_std": 0.6018249355256557, |
| "reward_before_mean": 0.40752510842867196, |
| "reward_before_std": 0.4566116305068135, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4376424718648195, |
| "reward_change_min": -0.6283221691846848, |
| "reward_change_std": 0.2420077919960022, |
| "reward_std": 0.601824939250946, |
| "rewards/accuracy_reward": 0.3541666679084301, |
| "rewards/cosine_scaled_reward": 0.05335840582847595, |
| "step": 124 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2841.687515258789, |
| "epoch": 0.14285714285714285, |
| "grad_norm": 0.057179540395736694, |
| "kl": 3.587547689676285e-05, |
| "lambda_div_used": 0.5674227699637413, |
| "learning_rate": 4.4113514698014953e-07, |
| "loss": 0.0092, |
| "reward": -0.09743379801511765, |
| "reward_after_mean": -0.09743379801511765, |
| "reward_after_std": 0.4501750059425831, |
| "reward_before_mean": 0.3561172317713499, |
| "reward_before_std": 0.3082335013896227, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.453551035374403, |
| "reward_change_min": -0.6685109585523605, |
| "reward_change_std": 0.25046134926378727, |
| "reward_std": 0.4501750282943249, |
| "rewards/accuracy_reward": 0.3125, |
| "rewards/cosine_scaled_reward": 0.043617233633995056, |
| "step": 125 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2814.3333892822266, |
| "epoch": 0.144, |
| "grad_norm": 0.06912728399038315, |
| "kl": 3.851950168609619e-05, |
| "lambda_div_used": 0.6011649072170258, |
| "learning_rate": 4.3353142970386557e-07, |
| "loss": 0.0079, |
| "reward": -0.2701383363455534, |
| "reward_after_mean": -0.2701383363455534, |
| "reward_after_std": 0.517305538058281, |
| "reward_before_mean": -0.00174633227288723, |
| "reward_before_std": 0.46759936958551407, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2683920059353113, |
| "reward_change_min": -0.42111679539084435, |
| "reward_change_std": 0.16303524654358625, |
| "reward_std": 0.5173055771738291, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.10591300774831325, |
| "step": 126 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3520.375, |
| "epoch": 0.14514285714285713, |
| "grad_norm": 0.05083320662379265, |
| "kl": 3.781914710998535e-05, |
| "lambda_div_used": 0.5308314934372902, |
| "learning_rate": 4.2596318988235037e-07, |
| "loss": 0.0094, |
| "reward": -0.5137354172766209, |
| "reward_after_mean": -0.5137354172766209, |
| "reward_after_std": 0.23104721494019032, |
| "reward_before_mean": -0.23012915067374706, |
| "reward_before_std": 0.13533379370346665, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28360624983906746, |
| "reward_change_min": -0.4074147306382656, |
| "reward_change_std": 0.14828919060528278, |
| "reward_std": 0.2310472223907709, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2301291599869728, |
| "step": 127 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2622.6041946411133, |
| "epoch": 0.1462857142857143, |
| "grad_norm": 0.07047850638628006, |
| "kl": 3.9443373680114746e-05, |
| "lambda_div_used": 0.5977307558059692, |
| "learning_rate": 4.1843273287476854e-07, |
| "loss": 0.0245, |
| "reward": 0.14617427624762058, |
| "reward_after_mean": 0.14617427624762058, |
| "reward_after_std": 0.5826170947402716, |
| "reward_before_mean": 0.6620007424353389, |
| "reward_before_std": 0.45144926803186536, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5158264562487602, |
| "reward_change_min": -0.7390152402222157, |
| "reward_change_std": 0.29064416885375977, |
| "reward_std": 0.5826170966029167, |
| "rewards/accuracy_reward": 0.41666666977107525, |
| "rewards/cosine_scaled_reward": 0.2453340534120798, |
| "step": 128 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3211.166702270508, |
| "epoch": 0.14742857142857144, |
| "grad_norm": 0.06908967345952988, |
| "kl": 4.245340824127197e-05, |
| "lambda_div_used": 0.579024501144886, |
| "learning_rate": 4.1094235253127374e-07, |
| "loss": -0.0068, |
| "reward": -0.41220802813768387, |
| "reward_after_mean": -0.41220802813768387, |
| "reward_after_std": 0.4235076569020748, |
| "reward_before_mean": -0.1792316001956351, |
| "reward_before_std": 0.3573149349540472, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2329764310270548, |
| "reward_change_min": -0.38812074810266495, |
| "reward_change_std": 0.13700235076248646, |
| "reward_std": 0.423507671803236, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.24173159897327423, |
| "step": 129 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3416.375030517578, |
| "epoch": 0.14857142857142858, |
| "grad_norm": 0.05305078998208046, |
| "kl": 3.8370490074157715e-05, |
| "lambda_div_used": 0.5585792362689972, |
| "learning_rate": 4.034943304942796e-07, |
| "loss": -0.0353, |
| "reward": -0.324653722345829, |
| "reward_after_mean": -0.324653722345829, |
| "reward_after_std": 0.30889566242694855, |
| "reward_before_mean": 0.0034640291705727577, |
| "reward_before_std": 0.2620142959058285, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.32811774499714375, |
| "reward_change_min": -0.4815611355006695, |
| "reward_change_std": 0.18737321346998215, |
| "reward_std": 0.30889566615223885, |
| "rewards/accuracy_reward": 0.1041666716337204, |
| "rewards/cosine_scaled_reward": -0.10070264618843794, |
| "step": 130 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2672.8541984558105, |
| "epoch": 0.14971428571428572, |
| "grad_norm": 0.08398960530757904, |
| "kl": 2.8876587748527527e-05, |
| "lambda_div_used": 0.6004800871014595, |
| "learning_rate": 3.9609093550344907e-07, |
| "loss": 0.0142, |
| "reward": 0.04825104773044586, |
| "reward_after_mean": 0.04825104773044586, |
| "reward_after_std": 0.537496130913496, |
| "reward_before_mean": 0.48470486514270306, |
| "reward_before_std": 0.4661959493532777, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.43645381927490234, |
| "reward_change_min": -0.650093249976635, |
| "reward_change_std": 0.2640222804620862, |
| "reward_std": 0.5374961327761412, |
| "rewards/accuracy_reward": 0.3750000074505806, |
| "rewards/cosine_scaled_reward": 0.10970486886799335, |
| "step": 131 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2745.8541870117188, |
| "epoch": 0.15085714285714286, |
| "grad_norm": 0.10217073559761047, |
| "kl": 3.91155481338501e-05, |
| "lambda_div_used": 0.5872602090239525, |
| "learning_rate": 3.8873442270461485e-07, |
| "loss": -0.0282, |
| "reward": -0.18162552546709776, |
| "reward_after_mean": -0.18162552546709776, |
| "reward_after_std": 0.4464551955461502, |
| "reward_before_mean": 0.14102918095886707, |
| "reward_before_std": 0.3960893382318318, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.32265469804406166, |
| "reward_change_min": -0.477376826107502, |
| "reward_change_std": 0.18945662677288055, |
| "reward_std": 0.4464551992714405, |
| "rewards/accuracy_reward": 0.22916667722165585, |
| "rewards/cosine_scaled_reward": -0.08813749923137948, |
| "step": 132 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3396.625, |
| "epoch": 0.152, |
| "grad_norm": 0.05338770151138306, |
| "kl": 4.1738152503967285e-05, |
| "lambda_div_used": 0.5715877488255501, |
| "learning_rate": 3.8142703296283953e-07, |
| "loss": -0.0083, |
| "reward": -0.3019852191209793, |
| "reward_after_mean": -0.3019852191209793, |
| "reward_after_std": 0.394625848159194, |
| "reward_before_mean": 0.015070955269038677, |
| "reward_before_std": 0.3251611590385437, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3170561697334051, |
| "reward_change_min": -0.5240310952067375, |
| "reward_change_std": 0.18560713436454535, |
| "reward_std": 0.39462585374712944, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.15159572381526232, |
| "step": 133 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2655.7708435058594, |
| "epoch": 0.15314285714285714, |
| "grad_norm": 0.08679015934467316, |
| "kl": 8.308887481689453e-05, |
| "lambda_div_used": 0.5542216002941132, |
| "learning_rate": 3.7417099217982686e-07, |
| "loss": 0.0208, |
| "reward": -0.32178156822919846, |
| "reward_after_mean": -0.32178156822919846, |
| "reward_after_std": 0.2957470379769802, |
| "reward_before_mean": 0.012201536446809769, |
| "reward_before_std": 0.24192149471491575, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3339830953627825, |
| "reward_change_min": -0.4839101508259773, |
| "reward_change_std": 0.18742438219487667, |
| "reward_std": 0.2957470417022705, |
| "rewards/accuracy_reward": 0.1041666716337204, |
| "rewards/cosine_scaled_reward": -0.09196514077484608, |
| "step": 134 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2014.9583587646484, |
| "epoch": 0.15428571428571428, |
| "grad_norm": 0.07728467881679535, |
| "kl": 2.7412548661231995e-05, |
| "lambda_div_used": 0.6218282878398895, |
| "learning_rate": 3.6696851061588994e-07, |
| "loss": 0.0739, |
| "reward": 0.15841423906385899, |
| "reward_after_mean": 0.15841423906385899, |
| "reward_after_std": 0.6740392297506332, |
| "reward_before_mean": 0.6407645158469677, |
| "reward_before_std": 0.5697397403419018, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.48235028609633446, |
| "reward_change_min": -0.7837426699697971, |
| "reward_change_std": 0.3034245353192091, |
| "reward_std": 0.6740392409265041, |
| "rewards/accuracy_reward": 0.4791666716337204, |
| "rewards/cosine_scaled_reward": 0.161597837228328, |
| "step": 135 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2602.416717529297, |
| "epoch": 0.15542857142857142, |
| "grad_norm": 0.06872163712978363, |
| "kl": 2.8759241104125977e-05, |
| "lambda_div_used": 0.6318690255284309, |
| "learning_rate": 3.5982178221668533e-07, |
| "loss": 0.0344, |
| "reward": 0.06710329907946289, |
| "reward_after_mean": 0.06710329907946289, |
| "reward_after_std": 0.6661366745829582, |
| "reward_before_mean": 0.454304663464427, |
| "reward_before_std": 0.6177845690399408, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.38720136508345604, |
| "reward_change_min": -0.6763367056846619, |
| "reward_change_std": 0.258380358107388, |
| "reward_std": 0.6661366857588291, |
| "rewards/accuracy_reward": 0.3541666716337204, |
| "rewards/cosine_scaled_reward": 0.10013799648731947, |
| "step": 136 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3239.666679382324, |
| "epoch": 0.15657142857142858, |
| "grad_norm": 0.0593891404569149, |
| "kl": 3.842264413833618e-05, |
| "lambda_div_used": 0.5779730677604675, |
| "learning_rate": 3.5273298394491515e-07, |
| "loss": 0.0124, |
| "reward": -0.3537064976990223, |
| "reward_after_mean": -0.3537064976990223, |
| "reward_after_std": 0.41078851372003555, |
| "reward_before_mean": -0.08445308171212673, |
| "reward_before_std": 0.35645375214517117, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2692534364759922, |
| "reward_change_min": -0.4356107972562313, |
| "reward_change_std": 0.15999021660536528, |
| "reward_std": 0.4107885267585516, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.16778641194105148, |
| "step": 137 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2760.687526702881, |
| "epoch": 0.15771428571428572, |
| "grad_norm": 0.0790812149643898, |
| "kl": 2.8595328330993652e-05, |
| "lambda_div_used": 0.5799111127853394, |
| "learning_rate": 3.45704275117204e-07, |
| "loss": -0.0362, |
| "reward": -0.27829512720927596, |
| "reward_after_mean": -0.27829512720927596, |
| "reward_after_std": 0.41231589019298553, |
| "reward_before_mean": 0.02187724970281124, |
| "reward_before_std": 0.362152349203825, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3001723885536194, |
| "reward_change_min": -0.45827802270650864, |
| "reward_change_std": 0.17282870691269636, |
| "reward_std": 0.412315895780921, |
| "rewards/accuracy_reward": 0.12500000558793545, |
| "rewards/cosine_scaled_reward": -0.10312275495380163, |
| "step": 138 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2984.4583587646484, |
| "epoch": 0.15885714285714286, |
| "grad_norm": 0.06229131296277046, |
| "kl": 3.9517879486083984e-05, |
| "lambda_div_used": 0.5838895812630653, |
| "learning_rate": 3.387377967463493e-07, |
| "loss": -0.0589, |
| "reward": -0.3416835393290967, |
| "reward_after_mean": -0.3416835393290967, |
| "reward_after_std": 0.4420221708714962, |
| "reward_before_mean": -0.08248884417116642, |
| "reward_before_std": 0.384817186743021, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.259194690734148, |
| "reward_change_min": -0.42579447478055954, |
| "reward_change_std": 0.15513746719807386, |
| "reward_std": 0.44202217273414135, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.16582217533141375, |
| "step": 139 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2881.1250228881836, |
| "epoch": 0.16, |
| "grad_norm": 0.071082703769207, |
| "kl": 4.310905933380127e-05, |
| "lambda_div_used": 0.5740059092640877, |
| "learning_rate": 3.3183567088914833e-07, |
| "loss": 0.0019, |
| "reward": -0.25879082828760147, |
| "reward_after_mean": -0.25879082828760147, |
| "reward_after_std": 0.4811950568109751, |
| "reward_before_mean": 0.09872006997466087, |
| "reward_before_std": 0.3372529884800315, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.35751091688871384, |
| "reward_change_min": -0.4904663935303688, |
| "reward_change_std": 0.18400432635098696, |
| "reward_std": 0.481195081025362, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.06794659808656434, |
| "step": 140 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3180.187515258789, |
| "epoch": 0.16114285714285714, |
| "grad_norm": 0.05953420698642731, |
| "kl": 3.790855407714844e-05, |
| "lambda_div_used": 0.5660471692681313, |
| "learning_rate": 3.250000000000001e-07, |
| "loss": -0.0557, |
| "reward": -0.3696493972092867, |
| "reward_after_mean": -0.3696493972092867, |
| "reward_after_std": 0.37020369805395603, |
| "reward_before_mean": -0.1031316639855504, |
| "reward_before_std": 0.300534725189209, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2665177509188652, |
| "reward_change_min": -0.3948013670742512, |
| "reward_change_std": 0.151769301854074, |
| "reward_std": 0.3702037110924721, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.20729831932112575, |
| "step": 141 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2633.291732788086, |
| "epoch": 0.16228571428571428, |
| "grad_norm": 0.06993869692087173, |
| "kl": 4.523620009422302e-05, |
| "lambda_div_used": 0.643665611743927, |
| "learning_rate": 3.182328662904756e-07, |
| "loss": 0.0607, |
| "reward": -0.11501466785557568, |
| "reward_after_mean": -0.11501466785557568, |
| "reward_after_std": 0.7061873953789473, |
| "reward_before_mean": 0.1335596082135453, |
| "reward_before_std": 0.6679177191108465, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2485742662101984, |
| "reward_change_min": -0.438314501196146, |
| "reward_change_std": 0.16557594947516918, |
| "reward_std": 0.7061873972415924, |
| "rewards/accuracy_reward": 0.18750000186264515, |
| "rewards/cosine_scaled_reward": -0.05394040001556277, |
| "step": 142 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2543.854202270508, |
| "epoch": 0.16342857142857142, |
| "grad_norm": 0.09050207585096359, |
| "kl": 4.5418739318847656e-05, |
| "lambda_div_used": 0.5568482205271721, |
| "learning_rate": 3.115363310950578e-07, |
| "loss": 0.0559, |
| "reward": -0.44561293721199036, |
| "reward_after_mean": -0.44561293721199036, |
| "reward_after_std": 0.34061737172305584, |
| "reward_before_mean": -0.17733613029122353, |
| "reward_before_std": 0.2554386807605624, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26827680319547653, |
| "reward_change_min": -0.39999108389019966, |
| "reward_change_std": 0.14368562400341034, |
| "reward_std": 0.3406173773109913, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.19816946546779945, |
| "step": 143 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3062.2291984558105, |
| "epoch": 0.16457142857142856, |
| "grad_norm": 0.08773668855428696, |
| "kl": 4.3526291847229004e-05, |
| "lambda_div_used": 0.5946916490793228, |
| "learning_rate": 3.0491243424323783e-07, |
| "loss": 0.0214, |
| "reward": -0.1852840557694435, |
| "reward_after_mean": -0.1852840557694435, |
| "reward_after_std": 0.4624195992946625, |
| "reward_before_mean": 0.13713636994361877, |
| "reward_before_std": 0.4314339701086283, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3224204182624817, |
| "reward_change_min": -0.502110667526722, |
| "reward_change_std": 0.19803205784410238, |
| "reward_std": 0.4624196030199528, |
| "rewards/accuracy_reward": 0.18750000558793545, |
| "rewards/cosine_scaled_reward": -0.05036364169791341, |
| "step": 144 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2343.708381652832, |
| "epoch": 0.1657142857142857, |
| "grad_norm": 0.09618457406759262, |
| "kl": 3.914535045623779e-05, |
| "lambda_div_used": 0.6065893918275833, |
| "learning_rate": 2.9836319343816397e-07, |
| "loss": 0.0239, |
| "reward": -0.08643046766519547, |
| "reward_after_mean": -0.08643046766519547, |
| "reward_after_std": 0.565411014482379, |
| "reward_before_mean": 0.2700228439643979, |
| "reward_before_std": 0.4942969996482134, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.35645330883562565, |
| "reward_change_min": -0.5621693357825279, |
| "reward_change_std": 0.21702316030859947, |
| "reward_std": 0.565411014482379, |
| "rewards/accuracy_reward": 0.29166666977107525, |
| "rewards/cosine_scaled_reward": -0.021643826737999916, |
| "step": 145 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2643.750030517578, |
| "epoch": 0.16685714285714287, |
| "grad_norm": 0.06749647855758667, |
| "kl": 3.0444934964179993e-05, |
| "lambda_div_used": 0.5759064182639122, |
| "learning_rate": 2.918906036420294e-07, |
| "loss": -0.0525, |
| "reward": -0.3986330684274435, |
| "reward_after_mean": -0.3986330684274435, |
| "reward_after_std": 0.4206914007663727, |
| "reward_before_mean": -0.1481835450977087, |
| "reward_before_std": 0.3465117560699582, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2504495307803154, |
| "reward_change_min": -0.3944549411535263, |
| "reward_change_std": 0.14348686579614878, |
| "reward_std": 0.42069140635430813, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.21068353950977325, |
| "step": 146 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3530.8958435058594, |
| "epoch": 0.168, |
| "grad_norm": 0.05082716792821884, |
| "kl": 4.493445158004761e-05, |
| "lambda_div_used": 0.6231074929237366, |
| "learning_rate": 2.854966364683872e-07, |
| "loss": 0.0064, |
| "reward": -0.17444764077663422, |
| "reward_after_mean": -0.17444764077663422, |
| "reward_after_std": 0.5966980569064617, |
| "reward_before_mean": 0.08711316343396902, |
| "reward_before_std": 0.5713155549019575, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26156080327928066, |
| "reward_change_min": -0.42396802455186844, |
| "reward_change_std": 0.16937424894422293, |
| "reward_std": 0.5966980736702681, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/cosine_scaled_reward": -0.12122016958892345, |
| "step": 147 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2726.0625228881836, |
| "epoch": 0.16914285714285715, |
| "grad_norm": 0.05811236426234245, |
| "kl": 2.9481947422027588e-05, |
| "lambda_div_used": 0.5576904863119125, |
| "learning_rate": 2.791832395815782e-07, |
| "loss": -0.0067, |
| "reward": -0.3092116080224514, |
| "reward_after_mean": -0.3092116080224514, |
| "reward_after_std": 0.3737166114151478, |
| "reward_before_mean": 0.04292176803573966, |
| "reward_before_std": 0.2566564744338393, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.35213335789740086, |
| "reward_change_min": -0.5105132721364498, |
| "reward_change_std": 0.18572800233960152, |
| "reward_std": 0.3737166281789541, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.10291159152984619, |
| "step": 148 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3018.4375228881836, |
| "epoch": 0.1702857142857143, |
| "grad_norm": 0.07346916198730469, |
| "kl": 3.395974636077881e-05, |
| "lambda_div_used": 0.5993617027997971, |
| "learning_rate": 2.729523361034538e-07, |
| "loss": 0.0316, |
| "reward": -0.23622393235564232, |
| "reward_after_mean": -0.23622393235564232, |
| "reward_after_std": 0.513044873252511, |
| "reward_before_mean": 0.05197374615818262, |
| "reward_before_std": 0.4586914679966867, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2881976682692766, |
| "reward_change_min": -0.4784680940210819, |
| "reward_change_std": 0.1794019928202033, |
| "reward_std": 0.5130448862910271, |
| "rewards/accuracy_reward": 0.14583333767950535, |
| "rewards/cosine_scaled_reward": -0.0938595961779356, |
| "step": 149 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2817.000030517578, |
| "epoch": 0.17142857142857143, |
| "grad_norm": 0.07110631465911865, |
| "kl": 4.3623149394989014e-05, |
| "lambda_div_used": 0.5749505385756493, |
| "learning_rate": 2.6680582402757324e-07, |
| "loss": 0.0143, |
| "reward": -0.4273622464388609, |
| "reward_after_mean": -0.4273622464388609, |
| "reward_after_std": 0.41951365023851395, |
| "reward_before_mean": -0.1882035918533802, |
| "reward_before_std": 0.34118228033185005, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.23915864899754524, |
| "reward_change_min": -0.3481520377099514, |
| "reward_change_std": 0.1273820260539651, |
| "reward_std": 0.4195136558264494, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.22987026162445545, |
| "step": 150 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2473.5208892822266, |
| "epoch": 0.17257142857142857, |
| "grad_norm": 0.06608390063047409, |
| "kl": 3.757700324058533e-05, |
| "lambda_div_used": 0.6444682851433754, |
| "learning_rate": 2.6074557564105724e-07, |
| "loss": 0.0674, |
| "reward": 0.013197226449847221, |
| "reward_after_mean": 0.013197226449847221, |
| "reward_after_std": 0.6638381816446781, |
| "reward_before_mean": 0.3273471943102777, |
| "reward_before_std": 0.6709178425371647, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.31414996832609177, |
| "reward_change_min": -0.5566080771386623, |
| "reward_change_std": 0.22394540812820196, |
| "reward_std": 0.6638382077217102, |
| "rewards/accuracy_reward": 0.31250000558793545, |
| "rewards/cosine_scaled_reward": 0.014847185462713242, |
| "step": 151 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2980.312530517578, |
| "epoch": 0.1737142857142857, |
| "grad_norm": 0.09729959070682526, |
| "kl": 3.904849290847778e-05, |
| "lambda_div_used": 0.5731147676706314, |
| "learning_rate": 2.547734369542718e-07, |
| "loss": 0.036, |
| "reward": -0.42251406982541084, |
| "reward_after_mean": -0.42251406982541084, |
| "reward_after_std": 0.4107065536081791, |
| "reward_before_mean": -0.17358114942908287, |
| "reward_before_std": 0.3313267915509641, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24893292412161827, |
| "reward_change_min": -0.37377795577049255, |
| "reward_change_std": 0.13549237046390772, |
| "reward_std": 0.4107065834105015, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.21524782478809357, |
| "step": 152 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2980.4167098999023, |
| "epoch": 0.17485714285714285, |
| "grad_norm": 0.07872413098812103, |
| "kl": 5.060434341430664e-05, |
| "lambda_div_used": 0.5996033921837807, |
| "learning_rate": 2.488912271385139e-07, |
| "loss": 0.0468, |
| "reward": -0.25326492823660374, |
| "reward_after_mean": -0.25326492823660374, |
| "reward_after_std": 0.5015600807964802, |
| "reward_before_mean": 0.03005093801766634, |
| "reward_before_std": 0.45544715132564306, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2833158541470766, |
| "reward_change_min": -0.490575835108757, |
| "reward_change_std": 0.17807927820831537, |
| "reward_std": 0.5015600919723511, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.09494908014312387, |
| "step": 153 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3400.6250610351562, |
| "epoch": 0.176, |
| "grad_norm": 0.044398024678230286, |
| "kl": 3.533065319061279e-05, |
| "lambda_div_used": 0.6407897770404816, |
| "learning_rate": 2.4310073797187573e-07, |
| "loss": 0.001, |
| "reward": -0.08846104983240366, |
| "reward_after_mean": -0.08846104983240366, |
| "reward_after_std": 0.6875216029584408, |
| "reward_before_mean": 0.1883750823326409, |
| "reward_before_std": 0.6521897967904806, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27683611772954464, |
| "reward_change_min": -0.45937369018793106, |
| "reward_change_std": 0.17437504325062037, |
| "reward_std": 0.6875216346234083, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": -0.019958254415541887, |
| "step": 154 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2453.5416717529297, |
| "epoch": 0.17714285714285713, |
| "grad_norm": 0.08257688581943512, |
| "kl": 4.6115368604660034e-05, |
| "lambda_div_used": 0.5891367048025131, |
| "learning_rate": 2.374037332934512e-07, |
| "loss": 0.0267, |
| "reward": -0.14017992746084929, |
| "reward_after_mean": -0.14017992746084929, |
| "reward_after_std": 0.5328723080456257, |
| "reward_before_mean": 0.24201755598187447, |
| "reward_before_std": 0.40873255487531424, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3821975253522396, |
| "reward_change_min": -0.5966118611395359, |
| "reward_change_std": 0.22055233176797628, |
| "reward_std": 0.532872324809432, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": 0.012850900180637836, |
| "step": 155 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3058.145835876465, |
| "epoch": 0.1782857142857143, |
| "grad_norm": 0.07657067477703094, |
| "kl": 3.8407742977142334e-05, |
| "lambda_div_used": 0.5792115926742554, |
| "learning_rate": 2.3180194846605364e-07, |
| "loss": 0.0192, |
| "reward": -0.35964071936905384, |
| "reward_after_mean": -0.35964071936905384, |
| "reward_after_std": 0.41963592916727066, |
| "reward_before_mean": -0.10375695489346981, |
| "reward_before_std": 0.36636115331202745, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2558837812393904, |
| "reward_change_min": -0.40037716925144196, |
| "reward_change_std": 0.15252912789583206, |
| "reward_std": 0.41963593289256096, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.18709028977900743, |
| "step": 156 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3067.0833435058594, |
| "epoch": 0.17942857142857144, |
| "grad_norm": 0.0602630190551281, |
| "kl": 4.166364669799805e-05, |
| "lambda_div_used": 0.578017845749855, |
| "learning_rate": 2.2629708984760706e-07, |
| "loss": -0.0024, |
| "reward": -0.20843622065149248, |
| "reward_after_mean": -0.20843622065149248, |
| "reward_after_std": 0.46523306891322136, |
| "reward_before_mean": 0.15985593758523464, |
| "reward_before_std": 0.35562971234321594, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3682921752333641, |
| "reward_change_min": -0.5581395737826824, |
| "reward_change_std": 0.20453235507011414, |
| "reward_std": 0.46523308381438255, |
| "rewards/accuracy_reward": 0.18750000186264515, |
| "rewards/cosine_scaled_reward": -0.027644065208733082, |
| "step": 157 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2439.8125381469727, |
| "epoch": 0.18057142857142858, |
| "grad_norm": 0.07962260395288467, |
| "kl": 3.724917769432068e-05, |
| "lambda_div_used": 0.5700756460428238, |
| "learning_rate": 2.2089083427137329e-07, |
| "loss": 0.0427, |
| "reward": -0.09846613928675652, |
| "reward_after_mean": -0.09846613928675652, |
| "reward_after_std": 0.50458899512887, |
| "reward_before_mean": 0.36336963158100843, |
| "reward_before_std": 0.31559942476451397, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.46183576062321663, |
| "reward_change_min": -0.6303520426154137, |
| "reward_change_std": 0.23663399182260036, |
| "reward_std": 0.5045890025794506, |
| "rewards/accuracy_reward": 0.2916666679084301, |
| "rewards/cosine_scaled_reward": 0.07170296460390091, |
| "step": 158 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3283.7916717529297, |
| "epoch": 0.18171428571428572, |
| "grad_norm": 0.05873558670282364, |
| "kl": 4.16487455368042e-05, |
| "lambda_div_used": 0.565159484744072, |
| "learning_rate": 2.1558482853517253e-07, |
| "loss": -0.0183, |
| "reward": -0.43611275404691696, |
| "reward_after_mean": -0.43611275404691696, |
| "reward_after_std": 0.34730882942676544, |
| "reward_before_mean": -0.18830033391714096, |
| "reward_before_std": 0.2990688029676676, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.247812420129776, |
| "reward_change_min": -0.4116964153945446, |
| "reward_change_std": 0.15130725782364607, |
| "reward_std": 0.34730884805321693, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.2508003171533346, |
| "step": 159 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3132.4167098999023, |
| "epoch": 0.18285714285714286, |
| "grad_norm": 0.06507622450590134, |
| "kl": 5.078315734863281e-05, |
| "lambda_div_used": 0.5871393531560898, |
| "learning_rate": 2.1038068889975259e-07, |
| "loss": 0.0371, |
| "reward": -0.25536114536225796, |
| "reward_after_mean": -0.25536114536225796, |
| "reward_after_std": 0.4553165938705206, |
| "reward_before_mean": 0.0424603084102273, |
| "reward_before_std": 0.3940334524959326, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2978214379400015, |
| "reward_change_min": -0.43723243474960327, |
| "reward_change_std": 0.16810003202408552, |
| "reward_std": 0.45531659945845604, |
| "rewards/accuracy_reward": 0.1458333395421505, |
| "rewards/cosine_scaled_reward": -0.10337302926927805, |
| "step": 160 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1814.145866394043, |
| "epoch": 0.184, |
| "grad_norm": 0.09199656546115875, |
| "kl": 3.68654727935791e-05, |
| "lambda_div_used": 0.6221684664487839, |
| "learning_rate": 2.0528000059645995e-07, |
| "loss": -0.1313, |
| "reward": -0.14752347487956285, |
| "reward_after_mean": -0.14752347487956285, |
| "reward_after_std": 0.6091006584465504, |
| "reward_before_mean": 0.1306287944316864, |
| "reward_before_std": 0.5693164113909006, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2781522646546364, |
| "reward_change_min": -0.4651326909661293, |
| "reward_change_std": 0.17978444695472717, |
| "reward_std": 0.6091006807982922, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.03603787627071142, |
| "step": 161 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3445.2291870117188, |
| "epoch": 0.18514285714285714, |
| "grad_norm": 0.054613932967185974, |
| "kl": 4.4696033000946045e-05, |
| "lambda_div_used": 0.6093570217490196, |
| "learning_rate": 2.0028431734436308e-07, |
| "loss": 0.0111, |
| "reward": -0.21708323806524277, |
| "reward_after_mean": -0.21708323806524277, |
| "reward_after_std": 0.5340255293995142, |
| "reward_before_mean": 0.05781930312514305, |
| "reward_before_std": 0.5134020620025694, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27490255795419216, |
| "reward_change_min": -0.5003471188247204, |
| "reward_change_std": 0.1919182576239109, |
| "reward_std": 0.5340255443006754, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.10884736478328705, |
| "step": 162 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2758.5833435058594, |
| "epoch": 0.18628571428571428, |
| "grad_norm": 0.06869502365589142, |
| "kl": 3.579258918762207e-05, |
| "lambda_div_used": 0.5997234806418419, |
| "learning_rate": 1.9539516087697517e-07, |
| "loss": 0.025, |
| "reward": -0.0028839372098445892, |
| "reward_after_mean": -0.0028839372098445892, |
| "reward_after_std": 0.5558424014598131, |
| "reward_before_mean": 0.42010904336348176, |
| "reward_before_std": 0.46184782730415463, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.42299298755824566, |
| "reward_change_min": -0.614623662084341, |
| "reward_change_std": 0.2444069180637598, |
| "reward_std": 0.5558424014598131, |
| "rewards/accuracy_reward": 0.33333334140479565, |
| "rewards/cosine_scaled_reward": 0.08677570521831512, |
| "step": 163 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2280.4583435058594, |
| "epoch": 0.18742857142857142, |
| "grad_norm": 0.08430740982294083, |
| "kl": 3.807246685028076e-05, |
| "lambda_div_used": 0.583802655339241, |
| "learning_rate": 1.9061402047871833e-07, |
| "loss": -0.0005, |
| "reward": -0.14323966577649117, |
| "reward_after_mean": -0.14323966577649117, |
| "reward_after_std": 0.48269340209662914, |
| "reward_before_mean": 0.24511565826833248, |
| "reward_before_std": 0.38433590345084667, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.38835531286895275, |
| "reward_change_min": -0.6137315705418587, |
| "reward_change_std": 0.2298044739291072, |
| "reward_std": 0.48269341699779034, |
| "rewards/accuracy_reward": 0.2500000037252903, |
| "rewards/cosine_scaled_reward": -0.0048843612894415855, |
| "step": 164 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3350.2083435058594, |
| "epoch": 0.18857142857142858, |
| "grad_norm": 0.06055794283747673, |
| "kl": 4.6700239181518555e-05, |
| "lambda_div_used": 0.5597026646137238, |
| "learning_rate": 1.8594235253127372e-07, |
| "loss": -0.0239, |
| "reward": -0.383526224642992, |
| "reward_after_mean": -0.383526224642992, |
| "reward_after_std": 0.3388876337558031, |
| "reward_before_mean": -0.09019916784018278, |
| "reward_before_std": 0.27148230001330376, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2933270577341318, |
| "reward_change_min": -0.44717802107334137, |
| "reward_change_std": 0.16905023716390133, |
| "reward_std": 0.3388876374810934, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.15269917901605368, |
| "step": 165 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2993.7709045410156, |
| "epoch": 0.18971428571428572, |
| "grad_norm": 0.07070616632699966, |
| "kl": 3.575533628463745e-05, |
| "lambda_div_used": 0.6159055009484291, |
| "learning_rate": 1.8138158006995363e-07, |
| "loss": -0.1171, |
| "reward": -0.20591574627906084, |
| "reward_after_mean": -0.20591574627906084, |
| "reward_after_std": 0.568760309368372, |
| "reward_before_mean": 0.07225125166587532, |
| "reward_before_std": 0.5325462874025106, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2781669981777668, |
| "reward_change_min": -0.4841184914112091, |
| "reward_change_std": 0.17965693771839142, |
| "reward_std": 0.5687603335827589, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.0735820853151381, |
| "step": 166 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2243.4583740234375, |
| "epoch": 0.19085714285714286, |
| "grad_norm": 0.08954203128814697, |
| "kl": 3.8199592381715775e-05, |
| "lambda_div_used": 0.5807118713855743, |
| "learning_rate": 1.7693309235023127e-07, |
| "loss": -0.0227, |
| "reward": -0.12234261445701122, |
| "reward_after_mean": -0.12234261445701122, |
| "reward_after_std": 0.46423870138823986, |
| "reward_before_mean": 0.28887630719691515, |
| "reward_before_std": 0.3718760753981769, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.41121890768408775, |
| "reward_change_min": -0.6246605962514877, |
| "reward_change_std": 0.24211041443049908, |
| "reward_std": 0.4642387069761753, |
| "rewards/accuracy_reward": 0.2291666679084301, |
| "rewards/cosine_scaled_reward": 0.0597096448764205, |
| "step": 167 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3159.604217529297, |
| "epoch": 0.192, |
| "grad_norm": 0.05736541375517845, |
| "kl": 4.6193599700927734e-05, |
| "lambda_div_used": 0.6025644987821579, |
| "learning_rate": 1.7259824442455923e-07, |
| "loss": 0.0616, |
| "reward": -0.1081604603677988, |
| "reward_after_mean": -0.1081604603677988, |
| "reward_after_std": 0.5043755322694778, |
| "reward_before_mean": 0.23938040900975466, |
| "reward_before_std": 0.4762058244086802, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.347540894523263, |
| "reward_change_min": -0.5797867476940155, |
| "reward_change_std": 0.22277529910206795, |
| "reward_std": 0.504375534132123, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/cosine_scaled_reward": 0.031047106254845858, |
| "step": 168 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2691.166717529297, |
| "epoch": 0.19314285714285714, |
| "grad_norm": 0.05840221792459488, |
| "kl": 3.322819247841835e-05, |
| "lambda_div_used": 0.6123679727315903, |
| "learning_rate": 1.6837835672960831e-07, |
| "loss": -0.0221, |
| "reward": 0.15223310515284538, |
| "reward_after_mean": 0.15223310515284538, |
| "reward_after_std": 0.589836286380887, |
| "reward_before_mean": 0.622445510700345, |
| "reward_before_std": 0.5133083704859018, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4702123887836933, |
| "reward_change_min": -0.686709500849247, |
| "reward_change_std": 0.27231256756931543, |
| "reward_std": 0.5898362882435322, |
| "rewards/accuracy_reward": 0.45833334140479565, |
| "rewards/cosine_scaled_reward": 0.1641121432185173, |
| "step": 169 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2737.8542137145996, |
| "epoch": 0.19428571428571428, |
| "grad_norm": 0.08408840745687485, |
| "kl": 2.9318034648895264e-05, |
| "lambda_div_used": 0.5516072362661362, |
| "learning_rate": 1.6427471468404952e-07, |
| "loss": -0.0223, |
| "reward": -0.22534361481666565, |
| "reward_after_mean": -0.22534361481666565, |
| "reward_after_std": 0.34615582413971424, |
| "reward_before_mean": 0.17829649709165096, |
| "reward_before_std": 0.231270051561296, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4036401268094778, |
| "reward_change_min": -0.5518503561615944, |
| "reward_change_std": 0.21789801493287086, |
| "reward_std": 0.34615583159029484, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/cosine_scaled_reward": -0.05087016709148884, |
| "step": 170 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2810.208366394043, |
| "epoch": 0.19542857142857142, |
| "grad_norm": 0.07381287217140198, |
| "kl": 3.468245267868042e-05, |
| "lambda_div_used": 0.6072571501135826, |
| "learning_rate": 1.6028856829700258e-07, |
| "loss": 0.0763, |
| "reward": -0.0880913995206356, |
| "reward_after_mean": -0.0880913995206356, |
| "reward_after_std": 0.4982736185193062, |
| "reward_before_mean": 0.2392272837460041, |
| "reward_before_std": 0.4966685324907303, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3273186720907688, |
| "reward_change_min": -0.5419761650264263, |
| "reward_change_std": 0.21660141553729773, |
| "reward_std": 0.4982736259698868, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/cosine_scaled_reward": 0.010060586035251617, |
| "step": 171 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2782.833381652832, |
| "epoch": 0.19657142857142856, |
| "grad_norm": 0.08758535981178284, |
| "kl": 4.4949352741241455e-05, |
| "lambda_div_used": 0.5883520841598511, |
| "learning_rate": 1.5642113178727193e-07, |
| "loss": 0.0173, |
| "reward": -0.0421207002364099, |
| "reward_after_mean": -0.0421207002364099, |
| "reward_after_std": 0.5838102325797081, |
| "reward_before_mean": 0.41242535319179296, |
| "reward_before_std": 0.4052962730638683, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.45454608276486397, |
| "reward_change_min": -0.6413811855018139, |
| "reward_change_std": 0.2437124690040946, |
| "reward_std": 0.5838102586567402, |
| "rewards/accuracy_reward": 0.3333333358168602, |
| "rewards/cosine_scaled_reward": 0.07909201784059405, |
| "step": 172 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2038.3125114440918, |
| "epoch": 0.1977142857142857, |
| "grad_norm": 0.10115820169448853, |
| "kl": 2.4806708097457886e-05, |
| "lambda_div_used": 0.5800783261656761, |
| "learning_rate": 1.5267358321348285e-07, |
| "loss": -0.0379, |
| "reward": -0.3465901352465153, |
| "reward_after_mean": -0.3465901352465153, |
| "reward_after_std": 0.41013904474675655, |
| "reward_before_mean": -0.07510924944654107, |
| "reward_before_std": 0.3657920900732279, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27148088812828064, |
| "reward_change_min": -0.41459682397544384, |
| "reward_change_std": 0.16112061124294996, |
| "reward_std": 0.4101390540599823, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.17927592061460018, |
| "step": 173 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2429.7083740234375, |
| "epoch": 0.19885714285714284, |
| "grad_norm": 0.08579805493354797, |
| "kl": 4.820525646209717e-05, |
| "lambda_div_used": 0.5971189364790916, |
| "learning_rate": 1.4904706411523448e-07, |
| "loss": -0.0681, |
| "reward": -0.16348301246762276, |
| "reward_after_mean": -0.16348301246762276, |
| "reward_after_std": 0.5588766317814589, |
| "reward_before_mean": 0.18685074103996158, |
| "reward_before_std": 0.44327013567090034, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3503337763249874, |
| "reward_change_min": -0.5374101847410202, |
| "reward_change_std": 0.19548403285443783, |
| "reward_std": 0.5588766410946846, |
| "rewards/accuracy_reward": 0.2083333358168602, |
| "rewards/cosine_scaled_reward": -0.02148258499801159, |
| "step": 174 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2815.1458587646484, |
| "epoch": 0.2, |
| "grad_norm": 0.06386198848485947, |
| "kl": 3.6529265344142914e-05, |
| "lambda_div_used": 0.5798816308379173, |
| "learning_rate": 1.4554267916537495e-07, |
| "loss": 0.0362, |
| "reward": -0.08284463733434677, |
| "reward_after_mean": -0.08284463733434677, |
| "reward_after_std": 0.46138195879757404, |
| "reward_before_mean": 0.33735317550599575, |
| "reward_before_std": 0.3691369164735079, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4201977960765362, |
| "reward_change_min": -0.615710511803627, |
| "reward_change_std": 0.2454231232404709, |
| "reward_std": 0.46138197369873524, |
| "rewards/accuracy_reward": 0.2708333358168602, |
| "rewards/cosine_scaled_reward": 0.06651980988681316, |
| "step": 175 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2699.916748046875, |
| "epoch": 0.20114285714285715, |
| "grad_norm": 0.1062018871307373, |
| "kl": 3.183633089065552e-05, |
| "lambda_div_used": 0.6652076914906502, |
| "learning_rate": 1.4216149583350755e-07, |
| "loss": -0.0096, |
| "reward": 0.01935443957336247, |
| "reward_after_mean": 0.01935443957336247, |
| "reward_after_std": 0.7740565538406372, |
| "reward_before_mean": 0.30571601539850235, |
| "reward_before_std": 0.7710783276706934, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2863615807145834, |
| "reward_change_min": -0.5516976863145828, |
| "reward_change_std": 0.20931501779705286, |
| "reward_std": 0.7740565687417984, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": 0.03488267329521477, |
| "step": 176 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2837.062530517578, |
| "epoch": 0.2022857142857143, |
| "grad_norm": 0.10419050604104996, |
| "kl": 5.3942203521728516e-05, |
| "lambda_div_used": 0.5891516581177711, |
| "learning_rate": 1.3890454406082956e-07, |
| "loss": -0.094, |
| "reward": -0.275299109518528, |
| "reward_after_mean": -0.275299109518528, |
| "reward_after_std": 0.4370743464678526, |
| "reward_before_mean": 0.019260672852396965, |
| "reward_before_std": 0.4135833829641342, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.294559795409441, |
| "reward_change_min": -0.49321193993091583, |
| "reward_change_std": 0.19294326566159725, |
| "reward_std": 0.43707435205578804, |
| "rewards/accuracy_reward": 0.125, |
| "rewards/cosine_scaled_reward": -0.10573932714760303, |
| "step": 177 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2239.645866394043, |
| "epoch": 0.20342857142857143, |
| "grad_norm": 0.08482369780540466, |
| "kl": 3.471970558166504e-05, |
| "lambda_div_used": 0.5743999034166336, |
| "learning_rate": 1.3577281594640182e-07, |
| "loss": -0.012, |
| "reward": -0.2907655192539096, |
| "reward_after_mean": -0.2907655192539096, |
| "reward_after_std": 0.3906351812183857, |
| "reward_before_mean": 0.020154590718448162, |
| "reward_before_std": 0.33835357427597046, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.31092010997235775, |
| "reward_change_min": -0.4689374789595604, |
| "reward_change_std": 0.18138453178107738, |
| "reward_std": 0.3906352035701275, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.08401207532733679, |
| "step": 178 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2847.187530517578, |
| "epoch": 0.20457142857142857, |
| "grad_norm": 0.06261342018842697, |
| "kl": 3.7359073758125305e-05, |
| "lambda_div_used": 0.5993921086192131, |
| "learning_rate": 1.3276726544494571e-07, |
| "loss": 0.0003, |
| "reward": -0.3251352347433567, |
| "reward_after_mean": -0.3251352347433567, |
| "reward_after_std": 0.507962841540575, |
| "reward_before_mean": -0.08432525303214788, |
| "reward_before_std": 0.45724861416965723, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2408099938184023, |
| "reward_change_min": -0.39610618352890015, |
| "reward_change_std": 0.14655023906379938, |
| "reward_std": 0.5079628489911556, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.18849190324544907, |
| "step": 179 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2329.562515258789, |
| "epoch": 0.2057142857142857, |
| "grad_norm": 0.10947566479444504, |
| "kl": 3.916025161743164e-05, |
| "lambda_div_used": 0.6209289953112602, |
| "learning_rate": 1.2988880807625927e-07, |
| "loss": -0.0732, |
| "reward": 0.0392537759616971, |
| "reward_after_mean": 0.0392537759616971, |
| "reward_after_std": 0.6374544408172369, |
| "reward_before_mean": 0.4333432329003699, |
| "reward_before_std": 0.5579905398190022, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3940894529223442, |
| "reward_change_min": -0.5780177563428879, |
| "reward_change_std": 0.229821746237576, |
| "reward_std": 0.6374544575810432, |
| "rewards/accuracy_reward": 0.3541666753590107, |
| "rewards/cosine_scaled_reward": 0.07917657308280468, |
| "step": 180 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3174.9166870117188, |
| "epoch": 0.20685714285714285, |
| "grad_norm": 0.0638444796204567, |
| "kl": 4.488229751586914e-05, |
| "lambda_div_used": 0.5625592544674873, |
| "learning_rate": 1.2713832064634125e-07, |
| "loss": -0.0055, |
| "reward": -0.3669877387583256, |
| "reward_after_mean": -0.3669877387583256, |
| "reward_after_std": 0.3254594895988703, |
| "reward_before_mean": -0.07308395206928253, |
| "reward_before_std": 0.2811311110854149, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29390377551317215, |
| "reward_change_min": -0.47421957924962044, |
| "reward_change_std": 0.1733461432158947, |
| "reward_std": 0.3254595026373863, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.15641729161143303, |
| "step": 181 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2200.5000228881836, |
| "epoch": 0.208, |
| "grad_norm": 0.07948590070009232, |
| "kl": 3.180652856826782e-05, |
| "lambda_div_used": 0.5802313759922981, |
| "learning_rate": 1.2451664098030743e-07, |
| "loss": -0.0177, |
| "reward": -0.33155214530415833, |
| "reward_after_mean": -0.33155214530415833, |
| "reward_after_std": 0.4093964695930481, |
| "reward_before_mean": -0.04757622070610523, |
| "reward_before_std": 0.362399042584002, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2839759271591902, |
| "reward_change_min": -0.4600135274231434, |
| "reward_change_std": 0.1762974951416254, |
| "reward_std": 0.40939648635685444, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.19340955652296543, |
| "step": 182 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1919.0000381469727, |
| "epoch": 0.20914285714285713, |
| "grad_norm": 0.09855654090642929, |
| "kl": 3.127008676528931e-05, |
| "lambda_div_used": 0.5876503959298134, |
| "learning_rate": 1.220245676671809e-07, |
| "loss": 0.0354, |
| "reward": -0.29830155335366726, |
| "reward_after_mean": -0.29830155335366726, |
| "reward_after_std": 0.461369052529335, |
| "reward_before_mean": -0.01361087104305625, |
| "reward_before_std": 0.4003021940588951, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28469069860875607, |
| "reward_change_min": -0.4400797598063946, |
| "reward_change_std": 0.1702390005812049, |
| "reward_std": 0.4613690562546253, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.11777753569185734, |
| "step": 183 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2771.0416870117188, |
| "epoch": 0.2102857142857143, |
| "grad_norm": 0.09264298528432846, |
| "kl": 4.6037137508392334e-05, |
| "lambda_div_used": 0.5391388088464737, |
| "learning_rate": 1.1966285981663407e-07, |
| "loss": -0.001, |
| "reward": -0.3703397810459137, |
| "reward_after_mean": -0.3703397810459137, |
| "reward_after_std": 0.2937673106789589, |
| "reward_before_mean": -0.011352727189660072, |
| "reward_before_std": 0.17188004031777382, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.35898703522980213, |
| "reward_change_min": -0.5206632278859615, |
| "reward_change_std": 0.18913730140775442, |
| "reward_std": 0.29376731254160404, |
| "rewards/accuracy_reward": 0.125, |
| "rewards/cosine_scaled_reward": -0.13635273277759552, |
| "step": 184 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2770.9792098999023, |
| "epoch": 0.21142857142857144, |
| "grad_norm": 0.09669750928878784, |
| "kl": 4.3720006942749023e-05, |
| "lambda_div_used": 0.5636111497879028, |
| "learning_rate": 1.1743223682775649e-07, |
| "loss": 0.0651, |
| "reward": -0.3112166179344058, |
| "reward_after_mean": -0.3112166179344058, |
| "reward_after_std": 0.3695005215704441, |
| "reward_before_mean": -0.0006957156583666801, |
| "reward_before_std": 0.2874382403679192, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3105209097266197, |
| "reward_change_min": -0.43808290362358093, |
| "reward_change_std": 0.17091987561434507, |
| "reward_std": 0.369500532746315, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.10486237239092588, |
| "step": 185 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2965.375015258789, |
| "epoch": 0.21257142857142858, |
| "grad_norm": 0.05594424530863762, |
| "kl": 3.8955360651016235e-05, |
| "lambda_div_used": 0.608224630355835, |
| "learning_rate": 1.1533337816991931e-07, |
| "loss": -0.0681, |
| "reward": -0.20297073479741812, |
| "reward_after_mean": -0.20297073479741812, |
| "reward_after_std": 0.5205403883010149, |
| "reward_before_mean": 0.07951527182012796, |
| "reward_before_std": 0.502502404153347, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.282486030831933, |
| "reward_change_min": -0.47177543863654137, |
| "reward_change_std": 0.18455488048493862, |
| "reward_std": 0.5205404032021761, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/cosine_scaled_reward": -0.08715140074491501, |
| "step": 186 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2639.291702270508, |
| "epoch": 0.21371428571428572, |
| "grad_norm": 0.08843009173870087, |
| "kl": 4.921853542327881e-05, |
| "lambda_div_used": 0.5908937901258469, |
| "learning_rate": 1.1336692317580158e-07, |
| "loss": -0.0048, |
| "reward": -0.25490788742899895, |
| "reward_after_mean": -0.25490788742899895, |
| "reward_after_std": 0.4807432759553194, |
| "reward_before_mean": 0.04253344633616507, |
| "reward_before_std": 0.4119319263845682, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29744134843349457, |
| "reward_change_min": -0.4517475329339504, |
| "reward_change_std": 0.16703799460083246, |
| "reward_std": 0.48074328526854515, |
| "rewards/accuracy_reward": 0.1458333395421505, |
| "rewards/cosine_scaled_reward": -0.10329987667500973, |
| "step": 187 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3541.9791870117188, |
| "epoch": 0.21485714285714286, |
| "grad_norm": 0.05120408907532692, |
| "kl": 4.652142524719238e-05, |
| "lambda_div_used": 0.5589732080698013, |
| "learning_rate": 1.1153347084664419e-07, |
| "loss": -0.0029, |
| "reward": -0.4253443591296673, |
| "reward_after_mean": -0.4253443591296673, |
| "reward_after_std": 0.3237063102424145, |
| "reward_before_mean": -0.15165027976036072, |
| "reward_before_std": 0.26862882915884256, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27369407564401627, |
| "reward_change_min": -0.437398936599493, |
| "reward_change_std": 0.16332428343594074, |
| "reward_std": 0.3237063158303499, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.2141502844169736, |
| "step": 188 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2377.2708892822266, |
| "epoch": 0.216, |
| "grad_norm": 0.07240846008062363, |
| "kl": 5.1606446504592896e-05, |
| "lambda_div_used": 0.5700332000851631, |
| "learning_rate": 1.0983357966978745e-07, |
| "loss": 0.0342, |
| "reward": -0.3627132559195161, |
| "reward_after_mean": -0.3627132559195161, |
| "reward_after_std": 0.4038649797439575, |
| "reward_before_mean": -0.08412502333521843, |
| "reward_before_std": 0.3190039964392781, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2785882242023945, |
| "reward_change_min": -0.4183661602437496, |
| "reward_change_std": 0.15456592850387096, |
| "reward_std": 0.4038649834692478, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.1466250205412507, |
| "step": 189 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3024.041717529297, |
| "epoch": 0.21714285714285714, |
| "grad_norm": 0.0596420094370842, |
| "kl": 2.85319983959198e-05, |
| "lambda_div_used": 0.5832700356841087, |
| "learning_rate": 1.0826776744855121e-07, |
| "loss": -0.0126, |
| "reward": -0.17394864186644554, |
| "reward_after_mean": -0.17394864186644554, |
| "reward_after_std": 0.471806388348341, |
| "reward_before_mean": 0.198049274738878, |
| "reward_before_std": 0.38090503215789795, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.37199792452156544, |
| "reward_change_min": -0.5356397405266762, |
| "reward_change_std": 0.2117150044068694, |
| "reward_std": 0.4718063995242119, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": -0.031117402017116547, |
| "step": 190 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2440.3541870117188, |
| "epoch": 0.21828571428571428, |
| "grad_norm": 0.0756516084074974, |
| "kl": 3.930646926164627e-05, |
| "lambda_div_used": 0.5715140625834465, |
| "learning_rate": 1.068365111445064e-07, |
| "loss": 0.0003, |
| "reward": -0.284542384557426, |
| "reward_after_mean": -0.284542384557426, |
| "reward_after_std": 0.447479585185647, |
| "reward_before_mean": 0.05547440081136301, |
| "reward_before_std": 0.3240885529667139, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34001679718494415, |
| "reward_change_min": -0.4750482700765133, |
| "reward_change_std": 0.1778492433950305, |
| "reward_std": 0.4474795889109373, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.11119226738810539, |
| "step": 191 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3487.6041870117188, |
| "epoch": 0.21942857142857142, |
| "grad_norm": 0.048071179538965225, |
| "kl": 3.250688314437866e-05, |
| "lambda_div_used": 0.5586921945214272, |
| "learning_rate": 1.0554024673218806e-07, |
| "loss": 0.0206, |
| "reward": -0.3871428966522217, |
| "reward_after_mean": -0.3871428966522217, |
| "reward_after_std": 0.33264124393463135, |
| "reward_before_mean": -0.11114806681871414, |
| "reward_before_std": 0.2715215114876628, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27599484845995903, |
| "reward_change_min": -0.40478406473994255, |
| "reward_change_std": 0.15969175938516855, |
| "reward_std": 0.33264124765992165, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.19448141008615494, |
| "step": 192 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2929.937545776367, |
| "epoch": 0.22057142857142858, |
| "grad_norm": 0.06499005109071732, |
| "kl": 3.759562969207764e-05, |
| "lambda_div_used": 0.6119573265314102, |
| "learning_rate": 1.0437936906629334e-07, |
| "loss": 0.0198, |
| "reward": -0.24959510192275047, |
| "reward_after_mean": -0.24959510192275047, |
| "reward_after_std": 0.5340033005923033, |
| "reward_before_mean": 0.009953722357749939, |
| "reward_before_std": 0.520604582503438, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25954881869256496, |
| "reward_change_min": -0.5182105302810669, |
| "reward_change_std": 0.186925214715302, |
| "reward_std": 0.5340033229440451, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.1358796115964651, |
| "step": 193 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3132.875030517578, |
| "epoch": 0.22171428571428572, |
| "grad_norm": 0.06673065572977066, |
| "kl": 4.443526268005371e-05, |
| "lambda_div_used": 0.6207702159881592, |
| "learning_rate": 1.0335423176140511e-07, |
| "loss": 0.0719, |
| "reward": 0.2215312235057354, |
| "reward_after_mean": 0.2215312235057354, |
| "reward_after_std": 0.6115883849561214, |
| "reward_before_mean": 0.7030764240771532, |
| "reward_before_std": 0.5582148376852274, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.48154521360993385, |
| "reward_change_min": -0.7418999075889587, |
| "reward_change_std": 0.29459451511502266, |
| "reward_std": 0.611588392406702, |
| "rewards/accuracy_reward": 0.4583333469927311, |
| "rewards/cosine_scaled_reward": 0.2447430812753737, |
| "step": 194 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2998.687530517578, |
| "epoch": 0.22285714285714286, |
| "grad_norm": 0.05692190304398537, |
| "kl": 4.340708255767822e-05, |
| "lambda_div_used": 0.5588866546750069, |
| "learning_rate": 1.0246514708427701e-07, |
| "loss": -0.0321, |
| "reward": -0.4250563494861126, |
| "reward_after_mean": -0.4250563494861126, |
| "reward_after_std": 0.3346366826444864, |
| "reward_before_mean": -0.16300050355494022, |
| "reward_before_std": 0.26834714552387595, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2620558477938175, |
| "reward_change_min": -0.4044179916381836, |
| "reward_change_std": 0.15184260439127684, |
| "reward_std": 0.3346366863697767, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.22550049051642418, |
| "step": 195 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3551.6041870117188, |
| "epoch": 0.224, |
| "grad_norm": 0.051314327865839005, |
| "kl": 4.8547983169555664e-05, |
| "lambda_div_used": 0.5673639252781868, |
| "learning_rate": 1.017123858587145e-07, |
| "loss": 0.0144, |
| "reward": -0.43661339208483696, |
| "reward_after_mean": -0.43661339208483696, |
| "reward_after_std": 0.3381017968058586, |
| "reward_before_mean": -0.18068170547485352, |
| "reward_before_std": 0.3060085875913501, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25593167915940285, |
| "reward_change_min": -0.436182364821434, |
| "reward_change_std": 0.16187659837305546, |
| "reward_std": 0.3381018117070198, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.24318172316998243, |
| "step": 196 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2789.125068664551, |
| "epoch": 0.22514285714285714, |
| "grad_norm": 0.08223231136798859, |
| "kl": 3.402773290872574e-05, |
| "lambda_div_used": 0.6149426028132439, |
| "learning_rate": 1.0109617738307911e-07, |
| "loss": 0.0526, |
| "reward": 0.08269345387816429, |
| "reward_after_mean": 0.08269345387816429, |
| "reward_after_std": 0.5922380294650793, |
| "reward_before_mean": 0.4997589197009802, |
| "reward_before_std": 0.5276397680863738, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4170654658228159, |
| "reward_change_min": -0.6376415118575096, |
| "reward_change_std": 0.24999654106795788, |
| "reward_std": 0.592238049954176, |
| "rewards/accuracy_reward": 0.39583334140479565, |
| "rewards/cosine_scaled_reward": 0.10392560251057148, |
| "step": 197 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2770.2708435058594, |
| "epoch": 0.22628571428571428, |
| "grad_norm": 0.07601747661828995, |
| "kl": 3.542006015777588e-05, |
| "lambda_div_used": 0.5800714045763016, |
| "learning_rate": 1.0061670936044178e-07, |
| "loss": -0.0178, |
| "reward": -0.24220025539398193, |
| "reward_after_mean": -0.24220025539398193, |
| "reward_after_std": 0.4058863651007414, |
| "reward_before_mean": 0.06675046496093273, |
| "reward_before_std": 0.36683204025030136, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.30895073898136616, |
| "reward_change_min": -0.4967211000621319, |
| "reward_change_std": 0.18911676667630672, |
| "reward_std": 0.40588637441396713, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.09991620294749737, |
| "step": 198 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3554.0416870117188, |
| "epoch": 0.22742857142857142, |
| "grad_norm": 0.04930044710636139, |
| "kl": 4.279613494873047e-05, |
| "lambda_div_used": 0.5560082867741585, |
| "learning_rate": 1.002741278414069e-07, |
| "loss": 0.0027, |
| "reward": -0.4229874052107334, |
| "reward_after_mean": -0.4229874052107334, |
| "reward_after_std": 0.31992789916694164, |
| "reward_before_mean": -0.14317837683483958, |
| "reward_before_std": 0.25433824164792895, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27980900928378105, |
| "reward_change_min": -0.4250169135630131, |
| "reward_change_std": 0.16277197189629078, |
| "reward_std": 0.31992790661752224, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.20567839220166206, |
| "step": 199 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2299.5209045410156, |
| "epoch": 0.22857142857142856, |
| "grad_norm": 0.0840829536318779, |
| "kl": 4.097074270248413e-05, |
| "lambda_div_used": 0.6226711273193359, |
| "learning_rate": 1.0006853717962393e-07, |
| "loss": 0.0203, |
| "reward": 0.062016794458031654, |
| "reward_after_mean": 0.062016794458031654, |
| "reward_after_std": 0.631916331127286, |
| "reward_before_mean": 0.4668920338153839, |
| "reward_before_std": 0.5674512181431055, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.40487524308264256, |
| "reward_change_min": -0.6527138948440552, |
| "reward_change_std": 0.250981617718935, |
| "reward_std": 0.6319163534790277, |
| "rewards/accuracy_reward": 0.35416667349636555, |
| "rewards/cosine_scaled_reward": 0.11272535985335708, |
| "step": 200 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2760.7084197998047, |
| "epoch": 0.2297142857142857, |
| "grad_norm": 0.06477575749158859, |
| "kl": 1.7711427062749863e-05, |
| "lambda_div_used": 0.6429438292980194, |
| "learning_rate": 1e-07, |
| "loss": -0.0022, |
| "reward": 0.15019571036100388, |
| "reward_after_mean": 0.15019571036100388, |
| "reward_after_std": 0.7328120246529579, |
| "reward_before_mean": 0.5535202682949603, |
| "reward_before_std": 0.6601617820560932, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4033245462924242, |
| "reward_change_min": -0.6303750872612, |
| "reward_change_std": 0.24585676938295364, |
| "reward_std": 0.7328120358288288, |
| "rewards/accuracy_reward": 0.41666667349636555, |
| "rewards/cosine_scaled_reward": 0.13685358315706253, |
| "step": 201 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2511.937511444092, |
| "epoch": 0.23085714285714284, |
| "grad_norm": 0.08345558494329453, |
| "kl": 4.0203332901000977e-05, |
| "lambda_div_used": 0.5811712816357613, |
| "learning_rate": 7.72273839962904e-07, |
| "loss": 0.0587, |
| "reward": 0.03902309015393257, |
| "reward_after_mean": 0.03902309015393257, |
| "reward_after_std": 0.5094954669475555, |
| "reward_before_mean": 0.5408022310584784, |
| "reward_before_std": 0.37269798293709755, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5017791502177715, |
| "reward_change_min": -0.7236880213022232, |
| "reward_change_std": 0.28125342447310686, |
| "reward_std": 0.5094954781234264, |
| "rewards/accuracy_reward": 0.3958333395421505, |
| "rewards/cosine_scaled_reward": 0.14496888127177954, |
| "step": 202 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3065.3333435058594, |
| "epoch": 0.232, |
| "grad_norm": 0.05931547284126282, |
| "kl": 5.182623863220215e-05, |
| "lambda_div_used": 0.5537669658660889, |
| "learning_rate": 7.695368466124296e-07, |
| "loss": 0.0447, |
| "reward": -0.3224771413952112, |
| "reward_after_mean": -0.3224771413952112, |
| "reward_after_std": 0.36307925172150135, |
| "reward_before_mean": 0.029698201455175877, |
| "reward_before_std": 0.23980092909187078, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3521753493696451, |
| "reward_change_min": -0.49568963050842285, |
| "reward_change_std": 0.18385440576821566, |
| "reward_std": 0.36307926289737225, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.1161351166665554, |
| "step": 203 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2511.833366394043, |
| "epoch": 0.23314285714285715, |
| "grad_norm": 0.07267706841230392, |
| "kl": 5.4508447647094727e-05, |
| "lambda_div_used": 0.5887176766991615, |
| "learning_rate": 7.667891533457718e-07, |
| "loss": 0.0023, |
| "reward": -0.23160922899842262, |
| "reward_after_mean": -0.23160922899842262, |
| "reward_after_std": 0.430733734741807, |
| "reward_before_mean": 0.06992286071181297, |
| "reward_before_std": 0.4116028640419245, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.301532082259655, |
| "reward_change_min": -0.49793890863657, |
| "reward_change_std": 0.19511268101632595, |
| "reward_std": 0.43073374405503273, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.07591048628091812, |
| "step": 204 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2967.1875915527344, |
| "epoch": 0.2342857142857143, |
| "grad_norm": 0.06776981800794601, |
| "kl": 4.439055919647217e-05, |
| "lambda_div_used": 0.653583250939846, |
| "learning_rate": 7.640308940816239e-07, |
| "loss": 0.1014, |
| "reward": -0.08090712130069733, |
| "reward_after_mean": -0.08090712130069733, |
| "reward_after_std": 0.7262772191315889, |
| "reward_before_mean": 0.17835846357047558, |
| "reward_before_std": 0.7181071005761623, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2592655848711729, |
| "reward_change_min": -0.4743462074548006, |
| "reward_change_std": 0.18475584778934717, |
| "reward_std": 0.7262772284448147, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/cosine_scaled_reward": -0.05080821271985769, |
| "step": 205 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3085.5208892822266, |
| "epoch": 0.23542857142857143, |
| "grad_norm": 0.06689873337745667, |
| "kl": 3.569573163986206e-05, |
| "lambda_div_used": 0.5862618833780289, |
| "learning_rate": 7.612622032536507e-07, |
| "loss": 0.0353, |
| "reward": -0.28365214727818966, |
| "reward_after_mean": -0.28365214727818966, |
| "reward_after_std": 0.44012872874736786, |
| "reward_before_mean": 0.004427256062626839, |
| "reward_before_std": 0.3988625044003129, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2880793921649456, |
| "reward_change_min": -0.49235254526138306, |
| "reward_change_std": 0.18395678792148829, |
| "reward_std": 0.44012873619794846, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.12057275045663118, |
| "step": 206 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3146.6458740234375, |
| "epoch": 0.23657142857142857, |
| "grad_norm": 0.06122450903058052, |
| "kl": 3.725104033946991e-05, |
| "lambda_div_used": 0.579835832118988, |
| "learning_rate": 7.584832158039378e-07, |
| "loss": 0.0182, |
| "reward": -0.28688727831467986, |
| "reward_after_mean": -0.28688727831467986, |
| "reward_after_std": 0.4150819983333349, |
| "reward_before_mean": 0.009919969365000725, |
| "reward_before_std": 0.3629662115126848, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29680725932121277, |
| "reward_change_min": -0.45919613167643547, |
| "reward_change_std": 0.17279880121350288, |
| "reward_std": 0.41508200392127037, |
| "rewards/accuracy_reward": 0.12500000558793545, |
| "rewards/cosine_scaled_reward": -0.11508002015762031, |
| "step": 207 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2780.291717529297, |
| "epoch": 0.2377142857142857, |
| "grad_norm": 0.06370716542005539, |
| "kl": 3.692507743835449e-05, |
| "lambda_div_used": 0.5807016789913177, |
| "learning_rate": 7.556940671764124e-07, |
| "loss": 0.0403, |
| "reward": -0.17094913870096207, |
| "reward_after_mean": -0.17094913870096207, |
| "reward_after_std": 0.46442117914557457, |
| "reward_before_mean": 0.20349296741187572, |
| "reward_before_std": 0.3671952560544014, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3744421415030956, |
| "reward_change_min": -0.5505415536463261, |
| "reward_change_std": 0.21210224367678165, |
| "reward_std": 0.46442119032144547, |
| "rewards/accuracy_reward": 0.20833333395421505, |
| "rewards/cosine_scaled_reward": -0.004840357229113579, |
| "step": 208 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2574.6042079925537, |
| "epoch": 0.23885714285714285, |
| "grad_norm": 0.08756011724472046, |
| "kl": 1.8547754734754562e-05, |
| "lambda_div_used": 0.6246868968009949, |
| "learning_rate": 7.528948933102438e-07, |
| "loss": 0.0837, |
| "reward": 0.028150439262390137, |
| "reward_after_mean": 0.028150439262390137, |
| "reward_after_std": 0.5636487938463688, |
| "reward_before_mean": 0.3838757500052452, |
| "reward_before_std": 0.576031070202589, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3557253200560808, |
| "reward_change_min": -0.5899170190095901, |
| "reward_change_std": 0.23946599010378122, |
| "reward_std": 0.5636488180607557, |
| "rewards/accuracy_reward": 0.3333333469927311, |
| "rewards/cosine_scaled_reward": 0.05054241791367531, |
| "step": 209 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2779.3333435058594, |
| "epoch": 0.24, |
| "grad_norm": 0.06111512333154678, |
| "kl": 3.0368566513061523e-05, |
| "lambda_div_used": 0.5897415727376938, |
| "learning_rate": 7.500858306332172e-07, |
| "loss": 0.0277, |
| "reward": -0.25500940857455134, |
| "reward_after_mean": -0.25500940857455134, |
| "reward_after_std": 0.47190882451832294, |
| "reward_before_mean": 0.03478116978658363, |
| "reward_before_std": 0.4098346810787916, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2897905595600605, |
| "reward_change_min": -0.42270801588892937, |
| "reward_change_std": 0.1620404813438654, |
| "reward_std": 0.4719088301062584, |
| "rewards/accuracy_reward": 0.1458333395421505, |
| "rewards/cosine_scaled_reward": -0.11105216934811324, |
| "step": 210 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2558.8541946411133, |
| "epoch": 0.24114285714285713, |
| "grad_norm": 0.06923027336597443, |
| "kl": 3.6522746086120605e-05, |
| "lambda_div_used": 0.5664671063423157, |
| "learning_rate": 7.472670160550848e-07, |
| "loss": 0.0729, |
| "reward": -0.06898702308535576, |
| "reward_after_mean": -0.06898702308535576, |
| "reward_after_std": 0.4354093614965677, |
| "reward_before_mean": 0.4075740482658148, |
| "reward_before_std": 0.30287738889455795, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4765610620379448, |
| "reward_change_min": -0.6883851811289787, |
| "reward_change_std": 0.262312775477767, |
| "reward_std": 0.4354093801230192, |
| "rewards/accuracy_reward": 0.3125, |
| "rewards/cosine_scaled_reward": 0.09507404454052448, |
| "step": 211 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2327.750015258789, |
| "epoch": 0.2422857142857143, |
| "grad_norm": 0.08028864860534668, |
| "kl": 2.8003007173538208e-05, |
| "lambda_div_used": 0.5712975114583969, |
| "learning_rate": 7.444385869608921e-07, |
| "loss": -0.0113, |
| "reward": -0.09517045877873898, |
| "reward_after_mean": -0.09517045877873898, |
| "reward_after_std": 0.49211281538009644, |
| "reward_before_mean": 0.3566615767776966, |
| "reward_before_std": 0.32028379291296005, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4518320318311453, |
| "reward_change_min": -0.6186705827713013, |
| "reward_change_std": 0.23167487233877182, |
| "reward_std": 0.4921128321439028, |
| "rewards/accuracy_reward": 0.2916666679084301, |
| "rewards/cosine_scaled_reward": 0.06499490264104679, |
| "step": 212 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2145.6041946411133, |
| "epoch": 0.24342857142857144, |
| "grad_norm": 0.09073800593614578, |
| "kl": 4.177819937467575e-05, |
| "lambda_div_used": 0.6168783828616142, |
| "learning_rate": 7.416006812042827e-07, |
| "loss": 0.027, |
| "reward": -0.08367926510982215, |
| "reward_after_mean": -0.08367926510982215, |
| "reward_after_std": 0.5790203902870417, |
| "reward_before_mean": 0.24448610469698906, |
| "reward_before_std": 0.5364782512187958, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.32816537097096443, |
| "reward_change_min": -0.5068525895476341, |
| "reward_change_std": 0.19713077135384083, |
| "reward_std": 0.579020407050848, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/cosine_scaled_reward": 0.036152773071080446, |
| "step": 213 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2982.6458587646484, |
| "epoch": 0.24457142857142858, |
| "grad_norm": 0.06497277319431305, |
| "kl": 3.772228956222534e-05, |
| "lambda_div_used": 0.6168569102883339, |
| "learning_rate": 7.387534371007797e-07, |
| "loss": 0.0247, |
| "reward": -0.10639690980315208, |
| "reward_after_mean": -0.10639690980315208, |
| "reward_after_std": 0.563305439427495, |
| "reward_before_mean": 0.21648684330284595, |
| "reward_before_std": 0.5375220291316509, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3228837437927723, |
| "reward_change_min": -0.5641361065208912, |
| "reward_change_std": 0.20987980626523495, |
| "reward_std": 0.5633054543286562, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": 0.00815348862670362, |
| "step": 214 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2382.2500610351562, |
| "epoch": 0.24571428571428572, |
| "grad_norm": 0.07877103984355927, |
| "kl": 4.485622048377991e-05, |
| "lambda_div_used": 0.5695274397730827, |
| "learning_rate": 7.358969934210438e-07, |
| "loss": 0.0351, |
| "reward": -0.39196348818950355, |
| "reward_after_mean": -0.39196348818950355, |
| "reward_after_std": 0.39862883277237415, |
| "reward_before_mean": -0.128106027841568, |
| "reward_before_std": 0.31178954988718033, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.263857439160347, |
| "reward_change_min": -0.38021961972117424, |
| "reward_change_std": 0.13994430005550385, |
| "reward_std": 0.39862884022295475, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.1697727027349174, |
| "step": 215 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2156.3958740234375, |
| "epoch": 0.24685714285714286, |
| "grad_norm": 0.11318857222795486, |
| "kl": 4.297494888305664e-05, |
| "lambda_div_used": 0.5965342745184898, |
| "learning_rate": 7.330314893841101e-07, |
| "loss": 0.0186, |
| "reward": -0.22045887634158134, |
| "reward_after_mean": -0.22045887634158134, |
| "reward_after_std": 0.4805217906832695, |
| "reward_before_mean": 0.08258800266776234, |
| "reward_before_std": 0.4415160808712244, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3030468635261059, |
| "reward_change_min": -0.4999500457197428, |
| "reward_change_std": 0.185057258233428, |
| "reward_std": 0.4805217981338501, |
| "rewards/accuracy_reward": 0.16666667349636555, |
| "rewards/cosine_scaled_reward": -0.08407866954803467, |
| "step": 216 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2763.520866394043, |
| "epoch": 0.248, |
| "grad_norm": 0.067295141518116, |
| "kl": 3.0644237995147705e-05, |
| "lambda_div_used": 0.6201670467853546, |
| "learning_rate": 7.301570646506027e-07, |
| "loss": 0.0139, |
| "reward": -0.19884846359491348, |
| "reward_after_mean": -0.19884846359491348, |
| "reward_after_std": 0.5856006741523743, |
| "reward_before_mean": 0.06158541014883667, |
| "reward_before_std": 0.5545480605214834, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2604338899254799, |
| "reward_change_min": -0.503634799271822, |
| "reward_change_std": 0.1784855630248785, |
| "reward_std": 0.5856006946414709, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/cosine_scaled_reward": -0.10508125275373459, |
| "step": 217 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2880.7083587646484, |
| "epoch": 0.24914285714285714, |
| "grad_norm": 0.07706139236688614, |
| "kl": 3.7044286727905273e-05, |
| "lambda_div_used": 0.6074161231517792, |
| "learning_rate": 7.27273859315928e-07, |
| "loss": -0.0223, |
| "reward": -0.19500153325498104, |
| "reward_after_mean": -0.19500153325498104, |
| "reward_after_std": 0.5332945492118597, |
| "reward_before_mean": 0.08328226953744888, |
| "reward_before_std": 0.5006468072533607, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27828381210565567, |
| "reward_change_min": -0.5113113783299923, |
| "reward_change_std": 0.18696410488337278, |
| "reward_std": 0.5332945715636015, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.08338439278304577, |
| "step": 218 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2352.645854949951, |
| "epoch": 0.2502857142857143, |
| "grad_norm": 0.10948827862739563, |
| "kl": 3.1441450119018555e-05, |
| "lambda_div_used": 0.5561031624674797, |
| "learning_rate": 7.243820139034464e-07, |
| "loss": -0.0096, |
| "reward": -0.029067307710647583, |
| "reward_after_mean": -0.029067307710647583, |
| "reward_after_std": 0.41141366213560104, |
| "reward_before_mean": 0.4911606255918741, |
| "reward_before_std": 0.2519808644428849, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5202279426157475, |
| "reward_change_min": -0.7093416638672352, |
| "reward_change_std": 0.27748389169573784, |
| "reward_std": 0.41141367703676224, |
| "rewards/accuracy_reward": 0.3541666716337204, |
| "rewards/cosine_scaled_reward": 0.13699393905699253, |
| "step": 219 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2635.5833435058594, |
| "epoch": 0.25142857142857145, |
| "grad_norm": 0.08914919197559357, |
| "kl": 3.5672448575496674e-05, |
| "lambda_div_used": 0.5308618620038033, |
| "learning_rate": 7.214816693576234e-07, |
| "loss": -0.0362, |
| "reward": -0.5146691724658012, |
| "reward_after_mean": -0.5146691724658012, |
| "reward_after_std": 0.23347610421478748, |
| "reward_before_mean": -0.2369950506836176, |
| "reward_before_std": 0.13545648753643036, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27767411433160305, |
| "reward_change_min": -0.3908136747777462, |
| "reward_change_std": 0.14405533485114574, |
| "reward_std": 0.23347610607743263, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2369950469583273, |
| "step": 220 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2279.562530517578, |
| "epoch": 0.25257142857142856, |
| "grad_norm": 0.0954662561416626, |
| "kl": 2.4788081645965576e-05, |
| "lambda_div_used": 0.5764833092689514, |
| "learning_rate": 7.185729670371604e-07, |
| "loss": -0.0168, |
| "reward": 0.023251693695783615, |
| "reward_after_mean": 0.023251693695783615, |
| "reward_after_std": 0.4863298423588276, |
| "reward_before_mean": 0.5212067291140556, |
| "reward_before_std": 0.34889572812244296, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.49795500561594963, |
| "reward_change_min": -0.7016883753240108, |
| "reward_change_std": 0.27911104913800955, |
| "reward_std": 0.48632985167205334, |
| "rewards/accuracy_reward": 0.4166666716337204, |
| "rewards/cosine_scaled_reward": 0.10454002395272255, |
| "step": 221 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2104.9583740234375, |
| "epoch": 0.2537142857142857, |
| "grad_norm": 0.0776790902018547, |
| "kl": 2.4726614356040955e-05, |
| "lambda_div_used": 0.5997953563928604, |
| "learning_rate": 7.156560487081051e-07, |
| "loss": -0.0148, |
| "reward": -0.04329463094472885, |
| "reward_after_mean": -0.04329463094472885, |
| "reward_after_std": 0.48106030002236366, |
| "reward_before_mean": 0.33003126084804535, |
| "reward_before_std": 0.46051184553653, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3733258917927742, |
| "reward_change_min": -0.5728014186024666, |
| "reward_change_std": 0.23287776950746775, |
| "reward_std": 0.48106031119823456, |
| "rewards/accuracy_reward": 0.2708333432674408, |
| "rewards/cosine_scaled_reward": 0.0591979268938303, |
| "step": 222 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2521.3333587646484, |
| "epoch": 0.25485714285714284, |
| "grad_norm": 0.07036174088716507, |
| "kl": 2.9304996132850647e-05, |
| "lambda_div_used": 0.5750112235546112, |
| "learning_rate": 7.127310565369415e-07, |
| "loss": 0.0216, |
| "reward": -0.1473498847335577, |
| "reward_after_mean": -0.1473498847335577, |
| "reward_after_std": 0.4359878208488226, |
| "reward_before_mean": 0.24288302287459373, |
| "reward_before_std": 0.3438769578933716, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.39023288898169994, |
| "reward_change_min": -0.588653527200222, |
| "reward_change_std": 0.22452317085117102, |
| "reward_std": 0.43598783388733864, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": -0.027950339019298553, |
| "step": 223 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3317.729217529297, |
| "epoch": 0.256, |
| "grad_norm": 0.05309538170695305, |
| "kl": 2.8050970286130905e-05, |
| "lambda_div_used": 0.5725216493010521, |
| "learning_rate": 7.097981330836616e-07, |
| "loss": 0.069, |
| "reward": -0.30242439545691013, |
| "reward_after_mean": -0.30242439545691013, |
| "reward_after_std": 0.38869454339146614, |
| "reward_before_mean": -0.000308917835354805, |
| "reward_before_std": 0.3251040354371071, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.30211549811065197, |
| "reward_change_min": -0.4553787522017956, |
| "reward_change_std": 0.171081081032753, |
| "reward_std": 0.3886945564299822, |
| "rewards/accuracy_reward": 0.12500000558793545, |
| "rewards/cosine_scaled_reward": -0.12530891119968146, |
| "step": 224 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3037.208335876465, |
| "epoch": 0.2571428571428571, |
| "grad_norm": 0.0768936425447464, |
| "kl": 4.080682992935181e-05, |
| "lambda_div_used": 0.5907088667154312, |
| "learning_rate": 7.068574212948169e-07, |
| "loss": 0.0169, |
| "reward": -0.28160014655441046, |
| "reward_after_mean": -0.28160014655441046, |
| "reward_after_std": 0.48472702503204346, |
| "reward_before_mean": -0.008234186680056155, |
| "reward_before_std": 0.41540220472961664, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27336596697568893, |
| "reward_change_min": -0.4061685614287853, |
| "reward_change_std": 0.15673903841525316, |
| "reward_std": 0.48472702503204346, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.13323417864739895, |
| "step": 225 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2861.416702270508, |
| "epoch": 0.2582857142857143, |
| "grad_norm": 0.07499676197767258, |
| "kl": 3.580749034881592e-05, |
| "lambda_div_used": 0.6208535805344582, |
| "learning_rate": 7.039090644965509e-07, |
| "loss": 0.0702, |
| "reward": -0.08291551470756531, |
| "reward_after_mean": -0.08291551470756531, |
| "reward_after_std": 0.5764818880707026, |
| "reward_before_mean": 0.23111886344850063, |
| "reward_before_std": 0.5593543313443661, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3140343725681305, |
| "reward_change_min": -0.5283008627593517, |
| "reward_change_std": 0.20683539099991322, |
| "reward_std": 0.5764818955212831, |
| "rewards/accuracy_reward": 0.27083334140479565, |
| "rewards/cosine_scaled_reward": -0.03971448230731767, |
| "step": 226 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1926.6875305175781, |
| "epoch": 0.25942857142857145, |
| "grad_norm": 0.10923109948635101, |
| "kl": 4.176795482635498e-05, |
| "lambda_div_used": 0.5834182798862457, |
| "learning_rate": 7.009532063876148e-07, |
| "loss": -0.0447, |
| "reward": -0.2778073139488697, |
| "reward_after_mean": -0.2778073139488697, |
| "reward_after_std": 0.4395454227924347, |
| "reward_before_mean": 0.009000460617244244, |
| "reward_before_std": 0.3901812704280019, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28680778108537197, |
| "reward_change_min": -0.45677450299263, |
| "reward_change_std": 0.18111994117498398, |
| "reward_std": 0.43954543210566044, |
| "rewards/accuracy_reward": 0.125, |
| "rewards/cosine_scaled_reward": -0.1159995449706912, |
| "step": 227 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2502.562545776367, |
| "epoch": 0.26057142857142856, |
| "grad_norm": 0.08182154595851898, |
| "kl": 1.8984079360961914e-05, |
| "lambda_div_used": 0.6025907471776009, |
| "learning_rate": 6.979899910323624e-07, |
| "loss": 0.074, |
| "reward": 0.03976001590490341, |
| "reward_after_mean": 0.03976001590490341, |
| "reward_after_std": 0.5421929359436035, |
| "reward_before_mean": 0.4711545445024967, |
| "reward_before_std": 0.4750876808539033, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4313945174217224, |
| "reward_change_min": -0.6774051003158092, |
| "reward_change_std": 0.2637105621397495, |
| "reward_std": 0.5421929433941841, |
| "rewards/accuracy_reward": 0.3541666753590107, |
| "rewards/cosine_scaled_reward": 0.11698784306645393, |
| "step": 228 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3196.3125228881836, |
| "epoch": 0.26171428571428573, |
| "grad_norm": 0.06398586928844452, |
| "kl": 4.0434300899505615e-05, |
| "lambda_div_used": 0.5669709742069244, |
| "learning_rate": 6.950195628537299e-07, |
| "loss": 0.0135, |
| "reward": -0.15348458290100098, |
| "reward_after_mean": -0.15348458290100098, |
| "reward_after_std": 0.42672324273735285, |
| "reward_before_mean": 0.26130594592541456, |
| "reward_before_std": 0.3047938751988113, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.41479056514799595, |
| "reward_change_min": -0.5674791261553764, |
| "reward_change_std": 0.22689451277256012, |
| "reward_std": 0.4267232706770301, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": 0.03213928057812154, |
| "step": 229 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3100.5000228881836, |
| "epoch": 0.26285714285714284, |
| "grad_norm": 0.05996527522802353, |
| "kl": 2.22623348236084e-05, |
| "lambda_div_used": 0.5811162814497948, |
| "learning_rate": 6.920420666261961e-07, |
| "loss": 0.0391, |
| "reward": -0.36068666726350784, |
| "reward_after_mean": -0.36068666726350784, |
| "reward_after_std": 0.43734684213995934, |
| "reward_before_mean": -0.08804995659738779, |
| "reward_before_std": 0.37305002473294735, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2726367134600878, |
| "reward_change_min": -0.4353605732321739, |
| "reward_change_std": 0.16089125256985426, |
| "reward_std": 0.43734684586524963, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.15054996183607727, |
| "step": 230 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2678.7083587646484, |
| "epoch": 0.264, |
| "grad_norm": 0.06770680844783783, |
| "kl": 2.5890767574310303e-05, |
| "lambda_div_used": 0.5817164853215218, |
| "learning_rate": 6.890576474687263e-07, |
| "loss": 0.0722, |
| "reward": -0.13625335786491632, |
| "reward_after_mean": -0.13625335786491632, |
| "reward_after_std": 0.46202925965189934, |
| "reward_before_mean": 0.25235490314662457, |
| "reward_before_std": 0.37311020120978355, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3886082824319601, |
| "reward_change_min": -0.6004670634865761, |
| "reward_change_std": 0.2275555245578289, |
| "reward_std": 0.4620292726904154, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": -0.018478410318493843, |
| "step": 231 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3258.625030517578, |
| "epoch": 0.2651428571428571, |
| "grad_norm": 0.05468503385782242, |
| "kl": 3.515183925628662e-05, |
| "lambda_div_used": 0.5800251811742783, |
| "learning_rate": 6.860664508377001e-07, |
| "loss": 0.0159, |
| "reward": -0.3428461756557226, |
| "reward_after_mean": -0.3428461756557226, |
| "reward_after_std": 0.4266065489500761, |
| "reward_before_mean": -0.07336848601698875, |
| "reward_before_std": 0.36755594704300165, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26947769708931446, |
| "reward_change_min": -0.43084757775068283, |
| "reward_change_std": 0.1597052849829197, |
| "reward_std": 0.4266065787523985, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.1567018236964941, |
| "step": 232 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2661.2292251586914, |
| "epoch": 0.2662857142857143, |
| "grad_norm": 0.08775883167982101, |
| "kl": 3.3989548683166504e-05, |
| "lambda_div_used": 0.6155472174286842, |
| "learning_rate": 6.83068622519821e-07, |
| "loss": -0.016, |
| "reward": -0.22876367531716824, |
| "reward_after_mean": -0.22876367531716824, |
| "reward_after_std": 0.5766322333365679, |
| "reward_before_mean": 0.027781556826084852, |
| "reward_before_std": 0.5290827043354511, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2565452288836241, |
| "reward_change_min": -0.4581022933125496, |
| "reward_change_std": 0.16627201065421104, |
| "reward_std": 0.5766322445124388, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.11805178504437208, |
| "step": 233 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2694.3125228881836, |
| "epoch": 0.2674285714285714, |
| "grad_norm": 0.11633959412574768, |
| "kl": 3.643333911895752e-05, |
| "lambda_div_used": 0.5539941042661667, |
| "learning_rate": 6.800643086250121e-07, |
| "loss": -0.0056, |
| "reward": -0.22800572216510773, |
| "reward_after_mean": -0.22800572216510773, |
| "reward_after_std": 0.3484720904380083, |
| "reward_before_mean": 0.16009100899100304, |
| "reward_before_std": 0.24206165876239538, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3880967255681753, |
| "reward_change_min": -0.5548957660794258, |
| "reward_change_std": 0.21049270872026682, |
| "reward_std": 0.34847209975123405, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/cosine_scaled_reward": -0.06907567009329796, |
| "step": 234 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2490.4375, |
| "epoch": 0.26857142857142857, |
| "grad_norm": 0.10378725826740265, |
| "kl": 2.8094742447137833e-05, |
| "lambda_div_used": 0.5782083421945572, |
| "learning_rate": 6.770536555792944e-07, |
| "loss": -0.0425, |
| "reward": 0.01590564101934433, |
| "reward_after_mean": 0.01590564101934433, |
| "reward_after_std": 0.5053408965468407, |
| "reward_before_mean": 0.49857149738818407, |
| "reward_before_std": 0.3586234971880913, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.48266585171222687, |
| "reward_change_min": -0.7199156694114208, |
| "reward_change_std": 0.2697788691148162, |
| "reward_std": 0.505340900272131, |
| "rewards/accuracy_reward": 0.3750000037252903, |
| "rewards/cosine_scaled_reward": 0.12357146013528109, |
| "step": 235 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2824.7917098999023, |
| "epoch": 0.26971428571428574, |
| "grad_norm": 0.06474865972995758, |
| "kl": 2.1502375602722168e-05, |
| "lambda_div_used": 0.5763456672430038, |
| "learning_rate": 6.740368101176495e-07, |
| "loss": 0.0011, |
| "reward": -0.3977004513144493, |
| "reward_after_mean": -0.3977004513144493, |
| "reward_after_std": 0.41489073634147644, |
| "reward_before_mean": -0.1460555698722601, |
| "reward_before_std": 0.347917802631855, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2516448702663183, |
| "reward_change_min": -0.42327918112277985, |
| "reward_change_std": 0.14836463797837496, |
| "reward_std": 0.4148907568305731, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.20855557406321168, |
| "step": 236 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2764.9583740234375, |
| "epoch": 0.27085714285714285, |
| "grad_norm": 0.06224860996007919, |
| "kl": 2.5756657123565674e-05, |
| "lambda_div_used": 0.5886275842785835, |
| "learning_rate": 6.710139192768694e-07, |
| "loss": -0.0228, |
| "reward": -0.1175742200575769, |
| "reward_after_mean": -0.1175742200575769, |
| "reward_after_std": 0.4582329224795103, |
| "reward_before_mean": 0.24255024455487728, |
| "reward_before_std": 0.40142686292529106, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36012447625398636, |
| "reward_change_min": -0.5257687419652939, |
| "reward_change_std": 0.20567627251148224, |
| "reward_std": 0.4582329299300909, |
| "rewards/accuracy_reward": 0.22916667722165585, |
| "rewards/cosine_scaled_reward": 0.013383567042183131, |
| "step": 237 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3222.479217529297, |
| "epoch": 0.272, |
| "grad_norm": 0.05841728672385216, |
| "kl": 3.546103835105896e-05, |
| "lambda_div_used": 0.6281886473298073, |
| "learning_rate": 6.679851303883891e-07, |
| "loss": 0.0655, |
| "reward": 0.044486068189144135, |
| "reward_after_mean": 0.044486068189144135, |
| "reward_after_std": 0.6025157757103443, |
| "reward_before_mean": 0.4036002438515425, |
| "reward_before_std": 0.6015807576477528, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.359114158898592, |
| "reward_change_min": -0.609935961663723, |
| "reward_change_std": 0.24452881701290607, |
| "reward_std": 0.6025157924741507, |
| "rewards/accuracy_reward": 0.3125000074505806, |
| "rewards/cosine_scaled_reward": 0.09110023453831673, |
| "step": 238 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1740.1666793823242, |
| "epoch": 0.27314285714285713, |
| "grad_norm": 0.09357193857431412, |
| "kl": 1.8790364265441895e-05, |
| "lambda_div_used": 0.5962883979082108, |
| "learning_rate": 6.649505910711058e-07, |
| "loss": 0.0055, |
| "reward": 0.07376761082559824, |
| "reward_after_mean": 0.07376761082559824, |
| "reward_after_std": 0.5891099888831377, |
| "reward_before_mean": 0.5742647312581539, |
| "reward_before_std": 0.44304153323173523, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5004971195012331, |
| "reward_change_min": -0.7436067499220371, |
| "reward_change_std": 0.2840570341795683, |
| "reward_std": 0.5891100075095892, |
| "rewards/accuracy_reward": 0.37500000186264515, |
| "rewards/cosine_scaled_reward": 0.19926471984945238, |
| "step": 239 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3087.416702270508, |
| "epoch": 0.2742857142857143, |
| "grad_norm": 0.06324295699596405, |
| "kl": 3.217160701751709e-05, |
| "lambda_div_used": 0.5339473709464073, |
| "learning_rate": 6.619104492241847e-07, |
| "loss": 0.0166, |
| "reward": -0.5358126908540726, |
| "reward_after_mean": -0.5358126908540726, |
| "reward_after_std": 0.23053276538848877, |
| "reward_before_mean": -0.2753155492246151, |
| "reward_before_std": 0.1490377252921462, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26049717888236046, |
| "reward_change_min": -0.3783254958689213, |
| "reward_change_std": 0.1383643699809909, |
| "reward_std": 0.23053277097642422, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2753155454993248, |
| "step": 240 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3419.625, |
| "epoch": 0.2754285714285714, |
| "grad_norm": 0.047396283596754074, |
| "kl": 3.555417060852051e-05, |
| "lambda_div_used": 0.5585425272583961, |
| "learning_rate": 6.588648530198504e-07, |
| "loss": 0.0027, |
| "reward": -0.4805446630343795, |
| "reward_after_mean": -0.4805446630343795, |
| "reward_after_std": 0.3355217073112726, |
| "reward_before_mean": -0.23604051489382982, |
| "reward_before_std": 0.2620125887915492, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2445041425526142, |
| "reward_change_min": -0.37724288925528526, |
| "reward_change_std": 0.13484715577214956, |
| "reward_std": 0.3355217222124338, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2568738413974643, |
| "step": 241 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2400.8750228881836, |
| "epoch": 0.2765714285714286, |
| "grad_norm": 0.08974741399288177, |
| "kl": 4.5746564865112305e-05, |
| "lambda_div_used": 0.5887154564261436, |
| "learning_rate": 6.558139508961654e-07, |
| "loss": -0.0452, |
| "reward": -0.2555653927847743, |
| "reward_after_mean": -0.2555653927847743, |
| "reward_after_std": 0.47001883387565613, |
| "reward_before_mean": 0.03157716616988182, |
| "reward_before_std": 0.4061046461574733, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28714255429804325, |
| "reward_change_min": -0.41172971203923225, |
| "reward_change_std": 0.16218022629618645, |
| "reward_std": 0.47001886926591396, |
| "rewards/accuracy_reward": 0.1458333395421505, |
| "rewards/cosine_scaled_reward": -0.1142561559099704, |
| "step": 242 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2828.541679382324, |
| "epoch": 0.2777142857142857, |
| "grad_norm": 0.07105688005685806, |
| "kl": 1.8571503460407257e-05, |
| "lambda_div_used": 0.5722818151116371, |
| "learning_rate": 6.527578915497951e-07, |
| "loss": 0.0039, |
| "reward": -0.1431608721613884, |
| "reward_after_mean": -0.1431608721613884, |
| "reward_after_std": 0.42974008433520794, |
| "reward_before_mean": 0.2529455106705427, |
| "reward_before_std": 0.3276430475525558, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.39610639959573746, |
| "reward_change_min": -0.5491434335708618, |
| "reward_change_std": 0.21527612209320068, |
| "reward_std": 0.42974009923636913, |
| "rewards/accuracy_reward": 0.25000000558793545, |
| "rewards/cosine_scaled_reward": 0.0029455198673531413, |
| "step": 243 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2882.291679382324, |
| "epoch": 0.27885714285714286, |
| "grad_norm": 0.0700126439332962, |
| "kl": 3.098323941230774e-05, |
| "lambda_div_used": 0.6161807402968407, |
| "learning_rate": 6.496968239287603e-07, |
| "loss": 0.0, |
| "reward": -0.06458889320492744, |
| "reward_after_mean": -0.06458889320492744, |
| "reward_after_std": 0.5576508566737175, |
| "reward_before_mean": 0.2605547234416008, |
| "reward_before_std": 0.5402053641155362, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.32514358684420586, |
| "reward_change_min": -0.5207333639264107, |
| "reward_change_std": 0.21059911139309406, |
| "reward_std": 0.5576508603990078, |
| "rewards/accuracy_reward": 0.27083334140479565, |
| "rewards/cosine_scaled_reward": -0.01027863984927535, |
| "step": 244 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2907.4583587646484, |
| "epoch": 0.28, |
| "grad_norm": 0.05816182866692543, |
| "kl": 3.5434961318969727e-05, |
| "lambda_div_used": 0.5615689232945442, |
| "learning_rate": 6.466308972251785e-07, |
| "loss": -0.0133, |
| "reward": -0.42641448229551315, |
| "reward_after_mean": -0.42641448229551315, |
| "reward_after_std": 0.3361923936754465, |
| "reward_before_mean": -0.15409247018396854, |
| "reward_before_std": 0.2813150165602565, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2723220158368349, |
| "reward_change_min": -0.445806298404932, |
| "reward_change_std": 0.1635848032310605, |
| "reward_std": 0.3361923974007368, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.19575914042070508, |
| "step": 245 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2680.7083587646484, |
| "epoch": 0.28114285714285714, |
| "grad_norm": 0.07198330760002136, |
| "kl": 1.764507032930851e-05, |
| "lambda_div_used": 0.5865297466516495, |
| "learning_rate": 6.435602608679916e-07, |
| "loss": 0.0347, |
| "reward": -0.19568787328898907, |
| "reward_after_mean": -0.19568787328898907, |
| "reward_after_std": 0.4485197216272354, |
| "reward_before_mean": 0.129365224391222, |
| "reward_before_std": 0.393932550214231, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.32505310885608196, |
| "reward_change_min": -0.4816659241914749, |
| "reward_change_std": 0.1897038472816348, |
| "reward_std": 0.44851974956691265, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": -0.0789681114256382, |
| "step": 246 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3076.0208435058594, |
| "epoch": 0.2822857142857143, |
| "grad_norm": 0.08006418496370316, |
| "kl": 4.15705144405365e-05, |
| "lambda_div_used": 0.5735789015889168, |
| "learning_rate": 6.404850645156841e-07, |
| "loss": 0.0199, |
| "reward": -0.3997113136574626, |
| "reward_after_mean": -0.3997113136574626, |
| "reward_after_std": 0.41638931445777416, |
| "reward_before_mean": -0.1426885835826397, |
| "reward_before_std": 0.3330965582281351, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2570227347314358, |
| "reward_change_min": -0.38052143156528473, |
| "reward_change_std": 0.13911327440291643, |
| "reward_std": 0.4163893237709999, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.18435525218956172, |
| "step": 247 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2145.0000228881836, |
| "epoch": 0.2834285714285714, |
| "grad_norm": 0.10052043944597244, |
| "kl": 3.3289194107055664e-05, |
| "lambda_div_used": 0.6171368733048439, |
| "learning_rate": 6.374054580489873e-07, |
| "loss": 0.0514, |
| "reward": 0.28496517799794674, |
| "reward_after_mean": 0.28496517799794674, |
| "reward_after_std": 0.6654367055743933, |
| "reward_before_mean": 0.83577667362988, |
| "reward_before_std": 0.5420532608404756, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5508115068078041, |
| "reward_change_min": -0.7994196638464928, |
| "reward_change_std": 0.3166588256135583, |
| "reward_std": 0.6654367428272963, |
| "rewards/accuracy_reward": 0.5000000093132257, |
| "rewards/cosine_scaled_reward": 0.33577666338533163, |
| "step": 248 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2076.937515258789, |
| "epoch": 0.2845714285714286, |
| "grad_norm": 0.0908605083823204, |
| "kl": 2.4942681193351746e-05, |
| "lambda_div_used": 0.5948201194405556, |
| "learning_rate": 6.343215915635761e-07, |
| "loss": 0.0219, |
| "reward": -0.003894178196787834, |
| "reward_after_mean": -0.003894178196787834, |
| "reward_after_std": 0.5242901761084795, |
| "reward_before_mean": 0.4153097262606025, |
| "reward_before_std": 0.43534869560971856, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.41920389235019684, |
| "reward_change_min": -0.5903440341353416, |
| "reward_change_std": 0.2394579891115427, |
| "reward_std": 0.524290194734931, |
| "rewards/accuracy_reward": 0.33333334140479565, |
| "rewards/cosine_scaled_reward": 0.08197638019919395, |
| "step": 249 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2605.0208587646484, |
| "epoch": 0.2857142857142857, |
| "grad_norm": 0.09027790278196335, |
| "kl": 3.804638981819153e-05, |
| "lambda_div_used": 0.5779460370540619, |
| "learning_rate": 6.31233615362752e-07, |
| "loss": -0.0589, |
| "reward": -0.2859988175332546, |
| "reward_after_mean": -0.2859988175332546, |
| "reward_after_std": 0.4080943390727043, |
| "reward_before_mean": 0.008792944252490997, |
| "reward_before_std": 0.35337654035538435, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2947917561978102, |
| "reward_change_min": -0.4365835040807724, |
| "reward_change_std": 0.16840537451207638, |
| "reward_std": 0.40809434466063976, |
| "rewards/accuracy_reward": 0.12500000558793545, |
| "rewards/cosine_scaled_reward": -0.11620705761015415, |
| "step": 250 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2368.9167137145996, |
| "epoch": 0.28685714285714287, |
| "grad_norm": 0.09640161693096161, |
| "kl": 2.7060508728027344e-05, |
| "lambda_div_used": 0.5987462773919106, |
| "learning_rate": 6.281416799501187e-07, |
| "loss": -0.0435, |
| "reward": -0.028017258271574974, |
| "reward_after_mean": -0.028017258271574974, |
| "reward_after_std": 0.5376799181103706, |
| "reward_before_mean": 0.38438196340575814, |
| "reward_before_std": 0.4528377316892147, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.41239920631051064, |
| "reward_change_min": -0.6119380593299866, |
| "reward_change_std": 0.23853347077965736, |
| "reward_std": 0.5376799292862415, |
| "rewards/accuracy_reward": 0.33333334140479565, |
| "rewards/cosine_scaled_reward": 0.05104860384017229, |
| "step": 251 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2798.5625228881836, |
| "epoch": 0.288, |
| "grad_norm": 0.06784425675868988, |
| "kl": 3.535952419042587e-05, |
| "lambda_div_used": 0.5580189973115921, |
| "learning_rate": 6.25045936022246e-07, |
| "loss": 0.0335, |
| "reward": -0.2931139934808016, |
| "reward_after_mean": -0.2931139934808016, |
| "reward_after_std": 0.38731373474001884, |
| "reward_before_mean": 0.06481979880481958, |
| "reward_before_std": 0.2658566879108548, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3579337988048792, |
| "reward_change_min": -0.5358714908361435, |
| "reward_change_std": 0.19932966493070126, |
| "reward_std": 0.3873137477785349, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.10184688493609428, |
| "step": 252 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2946.416679382324, |
| "epoch": 0.28914285714285715, |
| "grad_norm": 0.09991417825222015, |
| "kl": 3.4242868423461914e-05, |
| "lambda_div_used": 0.6016674339771271, |
| "learning_rate": 6.219465344613258e-07, |
| "loss": -0.0465, |
| "reward": -0.0356330550275743, |
| "reward_after_mean": -0.0356330550275743, |
| "reward_after_std": 0.5561915785074234, |
| "reward_before_mean": 0.353804474696517, |
| "reward_before_std": 0.4667428769171238, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.389437522739172, |
| "reward_change_min": -0.592089481651783, |
| "reward_change_std": 0.22914788126945496, |
| "reward_std": 0.5561915840953588, |
| "rewards/accuracy_reward": 0.31250000558793545, |
| "rewards/cosine_scaled_reward": 0.041304459096863866, |
| "step": 253 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2754.3541946411133, |
| "epoch": 0.29028571428571426, |
| "grad_norm": 0.08061102032661438, |
| "kl": 3.886967897415161e-05, |
| "lambda_div_used": 0.6251527816057205, |
| "learning_rate": 6.188436263278172e-07, |
| "loss": 0.0857, |
| "reward": -0.1664750911295414, |
| "reward_after_mean": -0.1664750911295414, |
| "reward_after_std": 0.599596256390214, |
| "reward_before_mean": 0.1026190984994173, |
| "reward_before_std": 0.5836509419605136, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2690941859036684, |
| "reward_change_min": -0.49960801005363464, |
| "reward_change_std": 0.19118957500904799, |
| "reward_std": 0.5995962955057621, |
| "rewards/accuracy_reward": 0.18750000186264515, |
| "rewards/cosine_scaled_reward": -0.08488089527236298, |
| "step": 254 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3190.083335876465, |
| "epoch": 0.2914285714285714, |
| "grad_norm": 0.08434556424617767, |
| "kl": 2.3322179913520813e-05, |
| "lambda_div_used": 0.5955284982919693, |
| "learning_rate": 6.157373628530852e-07, |
| "loss": -0.0125, |
| "reward": -0.3585695568472147, |
| "reward_after_mean": -0.3585695568472147, |
| "reward_after_std": 0.4996061436831951, |
| "reward_before_mean": -0.12086338270455599, |
| "reward_before_std": 0.4362535886466503, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2377061638981104, |
| "reward_change_min": -0.39885040931403637, |
| "reward_change_std": 0.1406589960679412, |
| "reward_std": 0.4996061585843563, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.20419671526178718, |
| "step": 255 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3160.8125610351562, |
| "epoch": 0.2925714285714286, |
| "grad_norm": 0.0576457642018795, |
| "kl": 3.143772482872009e-05, |
| "lambda_div_used": 0.639025017619133, |
| "learning_rate": 6.126278954320294e-07, |
| "loss": 0.0243, |
| "reward": -0.03833652473986149, |
| "reward_after_mean": -0.03833652473986149, |
| "reward_after_std": 0.6670235451310873, |
| "reward_before_mean": 0.2675688254312263, |
| "reward_before_std": 0.6473761759698391, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.30590534023940563, |
| "reward_change_min": -0.511355496942997, |
| "reward_change_std": 0.2077637044712901, |
| "reward_std": 0.6670235693454742, |
| "rewards/accuracy_reward": 0.2500000037252903, |
| "rewards/cosine_scaled_reward": 0.01756881456822157, |
| "step": 256 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3138.5000228881836, |
| "epoch": 0.2937142857142857, |
| "grad_norm": 0.05595054104924202, |
| "kl": 3.0346214771270752e-05, |
| "lambda_div_used": 0.6474356725811958, |
| "learning_rate": 6.095153756157051e-07, |
| "loss": 0.025, |
| "reward": -0.0769930558744818, |
| "reward_after_mean": -0.0769930558744818, |
| "reward_after_std": 0.7138958293944597, |
| "reward_before_mean": 0.18525638710707426, |
| "reward_before_std": 0.685269920155406, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26224944926798344, |
| "reward_change_min": -0.4810149297118187, |
| "reward_change_std": 0.18088674824684858, |
| "reward_std": 0.7138958312571049, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/cosine_scaled_reward": -0.04391028080135584, |
| "step": 257 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3473.187530517578, |
| "epoch": 0.2948571428571429, |
| "grad_norm": 0.05202889442443848, |
| "kl": 3.561750054359436e-05, |
| "lambda_div_used": 0.6007983982563019, |
| "learning_rate": 6.06399955103937e-07, |
| "loss": 0.0058, |
| "reward": -0.23316247668117285, |
| "reward_after_mean": -0.23316247668117285, |
| "reward_after_std": 0.5119184870272875, |
| "reward_before_mean": 0.04610642418265343, |
| "reward_before_std": 0.46757086645811796, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2792689222842455, |
| "reward_change_min": -0.4602624364197254, |
| "reward_change_std": 0.17828952055424452, |
| "reward_std": 0.5119185000658035, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.09972689533606172, |
| "step": 258 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2996.7500534057617, |
| "epoch": 0.296, |
| "grad_norm": 0.11353152990341187, |
| "kl": 4.0959566831588745e-05, |
| "lambda_div_used": 0.605031318962574, |
| "learning_rate": 6.032817857379256e-07, |
| "loss": -0.0572, |
| "reward": -0.24131755530834198, |
| "reward_after_mean": -0.24131755530834198, |
| "reward_after_std": 0.5208989772945642, |
| "reward_before_mean": 0.03226012596860528, |
| "reward_before_std": 0.4831458665430546, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27357766777276993, |
| "reward_change_min": -0.4753991588950157, |
| "reward_change_std": 0.1725512887351215, |
| "reward_std": 0.5208989884704351, |
| "rewards/accuracy_reward": 0.14583333767950535, |
| "rewards/cosine_scaled_reward": -0.1135732214897871, |
| "step": 259 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2184.8542098999023, |
| "epoch": 0.29714285714285715, |
| "grad_norm": 0.1352783739566803, |
| "kl": 2.3871660232543945e-05, |
| "lambda_div_used": 0.6604775562882423, |
| "learning_rate": 6.001610194928464e-07, |
| "loss": 0.0496, |
| "reward": 0.1305767484009266, |
| "reward_after_mean": 0.1305767484009266, |
| "reward_after_std": 0.7162795849144459, |
| "reward_before_mean": 0.48802967881783843, |
| "reward_before_std": 0.7462537074461579, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3574528992176056, |
| "reward_change_min": -0.6257632970809937, |
| "reward_change_std": 0.25755990110337734, |
| "reward_std": 0.7162796072661877, |
| "rewards/accuracy_reward": 0.39583334513008595, |
| "rewards/cosine_scaled_reward": 0.0921963145956397, |
| "step": 260 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3070.3125228881836, |
| "epoch": 0.29828571428571427, |
| "grad_norm": 0.06917808949947357, |
| "kl": 3.3020973205566406e-05, |
| "lambda_div_used": 0.5845921337604523, |
| "learning_rate": 5.97037808470444e-07, |
| "loss": 0.0106, |
| "reward": -0.3141371374949813, |
| "reward_after_mean": -0.3141371374949813, |
| "reward_after_std": 0.43580205366015434, |
| "reward_before_mean": -0.03358305338770151, |
| "reward_before_std": 0.38949463702738285, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28055410645902157, |
| "reward_change_min": -0.44674795493483543, |
| "reward_change_std": 0.16879158467054367, |
| "reward_std": 0.4358020592480898, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.11691637896001339, |
| "step": 261 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3258.3125, |
| "epoch": 0.29942857142857143, |
| "grad_norm": 0.06239396706223488, |
| "kl": 4.535168409347534e-05, |
| "lambda_div_used": 0.5396690741181374, |
| "learning_rate": 5.939123048916173e-07, |
| "loss": 0.0153, |
| "reward": -0.49852147325873375, |
| "reward_after_mean": -0.49852147325873375, |
| "reward_after_std": 0.23993146419525146, |
| "reward_before_mean": -0.21930878423154354, |
| "reward_before_std": 0.17422470543533564, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2792126890271902, |
| "reward_change_min": -0.4082174263894558, |
| "reward_change_std": 0.15261581167578697, |
| "reward_std": 0.2399314697831869, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.21930878423154354, |
| "step": 262 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2837.6666717529297, |
| "epoch": 0.30057142857142854, |
| "grad_norm": 0.07131204754114151, |
| "kl": 2.771243453025818e-05, |
| "lambda_div_used": 0.5753477811813354, |
| "learning_rate": 5.907846610890011e-07, |
| "loss": 0.0159, |
| "reward": -0.32574891671538353, |
| "reward_after_mean": -0.32574891671538353, |
| "reward_after_std": 0.40547293052077293, |
| "reward_before_mean": -0.04349888768047094, |
| "reward_before_std": 0.3437284992542118, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28225001133978367, |
| "reward_change_min": -0.423517182469368, |
| "reward_change_std": 0.16225541010499, |
| "reward_std": 0.4054729398339987, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.14766556210815907, |
| "step": 263 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2913.812515258789, |
| "epoch": 0.3017142857142857, |
| "grad_norm": 0.07276218384504318, |
| "kl": 3.5703182220458984e-05, |
| "lambda_div_used": 0.5857644975185394, |
| "learning_rate": 5.87655029499542e-07, |
| "loss": 0.039, |
| "reward": -0.19073306024074554, |
| "reward_after_mean": -0.19073306024074554, |
| "reward_after_std": 0.4183583725243807, |
| "reward_before_mean": 0.13579276762902737, |
| "reward_before_std": 0.39237749949097633, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.32652581483125687, |
| "reward_change_min": -0.512863963842392, |
| "reward_change_std": 0.20149299688637257, |
| "reward_std": 0.4183583725243807, |
| "rewards/accuracy_reward": 0.1875000074505806, |
| "rewards/cosine_scaled_reward": -0.051707250997424126, |
| "step": 264 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2092.333351135254, |
| "epoch": 0.3028571428571429, |
| "grad_norm": 0.09631343185901642, |
| "kl": 2.018176019191742e-05, |
| "lambda_div_used": 0.5927653685212135, |
| "learning_rate": 5.845235626570683e-07, |
| "loss": 0.0409, |
| "reward": -0.05978839658200741, |
| "reward_after_mean": -0.05978839658200741, |
| "reward_after_std": 0.5236263573169708, |
| "reward_before_mean": 0.3295632619410753, |
| "reward_before_std": 0.4253385625779629, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.38935166224837303, |
| "reward_change_min": -0.5855174511671066, |
| "reward_change_std": 0.22452097665518522, |
| "reward_std": 0.5236263833940029, |
| "rewards/accuracy_reward": 0.29166666977107525, |
| "rewards/cosine_scaled_reward": 0.037896597757935524, |
| "step": 265 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3210.8541717529297, |
| "epoch": 0.304, |
| "grad_norm": 0.05451129376888275, |
| "kl": 3.138929605484009e-05, |
| "lambda_div_used": 0.557657316327095, |
| "learning_rate": 5.813904131848564e-07, |
| "loss": -0.0024, |
| "reward": -0.3950183019042015, |
| "reward_after_mean": -0.3950183019042015, |
| "reward_after_std": 0.32154187746345997, |
| "reward_before_mean": -0.12019951082766056, |
| "reward_before_std": 0.2612606221809983, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2748187892138958, |
| "reward_change_min": -0.42080841958522797, |
| "reward_change_std": 0.158494733273983, |
| "reward_std": 0.321541890501976, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.20353284664452076, |
| "step": 266 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3039.916679382324, |
| "epoch": 0.30514285714285716, |
| "grad_norm": 0.06252676248550415, |
| "kl": 3.5075165214948356e-05, |
| "lambda_div_used": 0.5528770685195923, |
| "learning_rate": 5.78255733788191e-07, |
| "loss": -0.037, |
| "reward": -0.4460434205830097, |
| "reward_after_mean": -0.4460434205830097, |
| "reward_after_std": 0.3107005339115858, |
| "reward_before_mean": -0.1744655454531312, |
| "reward_before_std": 0.2377403611317277, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27157786674797535, |
| "reward_change_min": -0.42647456377744675, |
| "reward_change_std": 0.1541104121133685, |
| "reward_std": 0.3107005413621664, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.21613221243023872, |
| "step": 267 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2704.8333587646484, |
| "epoch": 0.3062857142857143, |
| "grad_norm": 0.08417963981628418, |
| "kl": 4.7713518142700195e-05, |
| "lambda_div_used": 0.6233388632535934, |
| "learning_rate": 5.751196772469237e-07, |
| "loss": 0.0822, |
| "reward": -0.21941478177905083, |
| "reward_after_mean": -0.21941478177905083, |
| "reward_after_std": 0.6098091676831245, |
| "reward_before_mean": 0.03345827816519886, |
| "reward_before_std": 0.5743174999952316, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25287305377423763, |
| "reward_change_min": -0.45724478363990784, |
| "reward_change_std": 0.1690685572102666, |
| "reward_std": 0.6098091900348663, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.11237506754696369, |
| "step": 268 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3165.9791717529297, |
| "epoch": 0.30742857142857144, |
| "grad_norm": 0.06371315568685532, |
| "kl": 2.181902527809143e-05, |
| "lambda_div_used": 0.5798910111188889, |
| "learning_rate": 5.71982396408026e-07, |
| "loss": 0.0477, |
| "reward": -0.38025568798184395, |
| "reward_after_mean": -0.38025568798184395, |
| "reward_after_std": 0.4126162938773632, |
| "reward_before_mean": -0.13023450784385204, |
| "reward_before_std": 0.3629760518670082, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2500211838632822, |
| "reward_change_min": -0.40024495497345924, |
| "reward_change_std": 0.15232266392558813, |
| "reward_std": 0.4126163087785244, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.2135678417980671, |
| "step": 269 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3056.875045776367, |
| "epoch": 0.30857142857142855, |
| "grad_norm": 0.05920446664094925, |
| "kl": 2.387911081314087e-05, |
| "lambda_div_used": 0.6503699943423271, |
| "learning_rate": 5.688440441781398e-07, |
| "loss": 0.0, |
| "reward": -0.004786740057170391, |
| "reward_after_mean": -0.004786740057170391, |
| "reward_after_std": 0.6943319551646709, |
| "reward_before_mean": 0.292231020051986, |
| "reward_before_std": 0.7036092299968004, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2970177587121725, |
| "reward_change_min": -0.5634518079459667, |
| "reward_change_std": 0.21872046310454607, |
| "reward_std": 0.6943319924175739, |
| "rewards/accuracy_reward": 0.27083333767950535, |
| "rewards/cosine_scaled_reward": 0.021397670439910144, |
| "step": 270 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2221.3958702087402, |
| "epoch": 0.3097142857142857, |
| "grad_norm": 0.09025074541568756, |
| "kl": 6.723217666149139e-06, |
| "lambda_div_used": 0.640851192176342, |
| "learning_rate": 5.657047735161255e-07, |
| "loss": 0.0064, |
| "reward": 0.11755780689418316, |
| "reward_after_mean": 0.11755780689418316, |
| "reward_after_std": 0.6356876939535141, |
| "reward_before_mean": 0.4748641401529312, |
| "reward_before_std": 0.654548792168498, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.35730634443461895, |
| "reward_change_min": -0.5644437614828348, |
| "reward_change_std": 0.23989163804799318, |
| "reward_std": 0.6356877088546753, |
| "rewards/accuracy_reward": 0.39583334885537624, |
| "rewards/cosine_scaled_reward": 0.07903079688549042, |
| "step": 271 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2814.5208740234375, |
| "epoch": 0.31085714285714283, |
| "grad_norm": 0.07214022427797318, |
| "kl": 2.553686499595642e-05, |
| "lambda_div_used": 0.5532237812876701, |
| "learning_rate": 5.625647374256061e-07, |
| "loss": -0.023, |
| "reward": -0.33643968403339386, |
| "reward_after_mean": -0.33643968403339386, |
| "reward_after_std": 0.35168860107660294, |
| "reward_before_mean": 0.01186647079885006, |
| "reward_before_std": 0.2353415172547102, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34830615669488907, |
| "reward_change_min": -0.5079392194747925, |
| "reward_change_std": 0.1846030419692397, |
| "reward_std": 0.351688614115119, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.13396686874330044, |
| "step": 272 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2673.937515258789, |
| "epoch": 0.312, |
| "grad_norm": 0.067450150847435, |
| "kl": 2.347538247704506e-05, |
| "lambda_div_used": 0.5976931154727936, |
| "learning_rate": 5.594240889475106e-07, |
| "loss": 0.0439, |
| "reward": 0.060379184782505035, |
| "reward_after_mean": 0.060379184782505035, |
| "reward_after_std": 0.5129444599151611, |
| "reward_before_mean": 0.4946505483239889, |
| "reward_before_std": 0.44454328902065754, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4342713989317417, |
| "reward_change_min": -0.6668536812067032, |
| "reward_change_std": 0.25412870943546295, |
| "reward_std": 0.5129444822669029, |
| "rewards/accuracy_reward": 0.3750000111758709, |
| "rewards/cosine_scaled_reward": 0.1196505706757307, |
| "step": 273 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1866.4375076293945, |
| "epoch": 0.31314285714285717, |
| "grad_norm": 0.11823767423629761, |
| "kl": 2.6108697056770325e-05, |
| "lambda_div_used": 0.6173145174980164, |
| "learning_rate": 5.562829811526154e-07, |
| "loss": -0.0666, |
| "reward": 0.20245935022830963, |
| "reward_after_mean": 0.20245935022830963, |
| "reward_after_std": 0.6078098546713591, |
| "reward_before_mean": 0.6919816123554483, |
| "reward_before_std": 0.5468513960950077, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4895222559571266, |
| "reward_change_min": -0.7379350513219833, |
| "reward_change_std": 0.3024911228567362, |
| "reward_std": 0.6078098695725203, |
| "rewards/accuracy_reward": 0.4583333432674408, |
| "rewards/cosine_scaled_reward": 0.2336482839891687, |
| "step": 274 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2334.0416870117188, |
| "epoch": 0.3142857142857143, |
| "grad_norm": 0.07390157133340836, |
| "kl": 2.975761890411377e-05, |
| "lambda_div_used": 0.5953021869063377, |
| "learning_rate": 5.531415671340826e-07, |
| "loss": 0.0067, |
| "reward": -0.02193348854780197, |
| "reward_after_mean": -0.02193348854780197, |
| "reward_after_std": 0.5273136273026466, |
| "reward_before_mean": 0.3926870714276447, |
| "reward_before_std": 0.4364564120769501, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4146205447614193, |
| "reward_change_min": -0.6028474234044552, |
| "reward_change_std": 0.23770872876048088, |
| "reward_std": 0.5273136328905821, |
| "rewards/accuracy_reward": 0.31250000558793545, |
| "rewards/cosine_scaled_reward": 0.08018706925213337, |
| "step": 275 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2735.6458740234375, |
| "epoch": 0.31542857142857145, |
| "grad_norm": 0.07891591638326645, |
| "kl": 3.522634506225586e-05, |
| "lambda_div_used": 0.6019129753112793, |
| "learning_rate": 5.5e-07, |
| "loss": -0.0433, |
| "reward": 0.016597013920545578, |
| "reward_after_mean": 0.016597013920545578, |
| "reward_after_std": 0.5496951006352901, |
| "reward_before_mean": 0.43198024667799473, |
| "reward_before_std": 0.4716393407434225, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4153832569718361, |
| "reward_change_min": -0.6486029289662838, |
| "reward_change_std": 0.25036295782774687, |
| "reward_std": 0.5496951248496771, |
| "rewards/accuracy_reward": 0.3333333395421505, |
| "rewards/cosine_scaled_reward": 0.09864690899848938, |
| "step": 276 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2435.437530517578, |
| "epoch": 0.31657142857142856, |
| "grad_norm": 0.08397295325994492, |
| "kl": 2.7611851692199707e-05, |
| "lambda_div_used": 0.6150463595986366, |
| "learning_rate": 5.468584328659172e-07, |
| "loss": 0.0536, |
| "reward": -0.049143560230731964, |
| "reward_after_mean": -0.049143560230731964, |
| "reward_after_std": 0.5724204778671265, |
| "reward_before_mean": 0.29686027206480503, |
| "reward_before_std": 0.5315626971423626, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34600381925702095, |
| "reward_change_min": -0.5762605480849743, |
| "reward_change_std": 0.21639508474618196, |
| "reward_std": 0.5724204815924168, |
| "rewards/accuracy_reward": 0.27083333767950535, |
| "rewards/cosine_scaled_reward": 0.02602693811058998, |
| "step": 277 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2029.708381652832, |
| "epoch": 0.3177142857142857, |
| "grad_norm": 0.11247767508029938, |
| "kl": 3.094971179962158e-05, |
| "lambda_div_used": 0.6170216798782349, |
| "learning_rate": 5.437170188473847e-07, |
| "loss": 0.002, |
| "reward": 0.056462954729795456, |
| "reward_after_mean": 0.056462954729795456, |
| "reward_after_std": 0.6216260213404894, |
| "reward_before_mean": 0.48691817931830883, |
| "reward_before_std": 0.5461275167763233, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4304552264511585, |
| "reward_change_min": -0.7186719551682472, |
| "reward_change_std": 0.27375681325793266, |
| "reward_std": 0.6216260306537151, |
| "rewards/accuracy_reward": 0.3541666716337204, |
| "rewards/cosine_scaled_reward": 0.13275149278342724, |
| "step": 278 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3158.6875228881836, |
| "epoch": 0.31885714285714284, |
| "grad_norm": 0.06945524364709854, |
| "kl": 3.522634506225586e-05, |
| "lambda_div_used": 0.5564287528395653, |
| "learning_rate": 5.405759110524894e-07, |
| "loss": -0.0272, |
| "reward": -0.43442236818373203, |
| "reward_after_mean": -0.43442236818373203, |
| "reward_after_std": 0.32187592424452305, |
| "reward_before_mean": -0.1544840056449175, |
| "reward_before_std": 0.25075172632932663, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27993838116526604, |
| "reward_change_min": -0.41151439025998116, |
| "reward_change_std": 0.15283891931176186, |
| "reward_std": 0.32187593914568424, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.17531732900533825, |
| "step": 279 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2157.833335876465, |
| "epoch": 0.32, |
| "grad_norm": 0.14424921572208405, |
| "kl": 4.202499985694885e-05, |
| "lambda_div_used": 0.630903884768486, |
| "learning_rate": 5.37435262574394e-07, |
| "loss": -0.0339, |
| "reward": -0.14438428170979023, |
| "reward_after_mean": -0.14438428170979023, |
| "reward_after_std": 0.6414528246968985, |
| "reward_before_mean": 0.1313652544049546, |
| "reward_before_std": 0.6011563409119844, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.275749534368515, |
| "reward_change_min": -0.4570390097796917, |
| "reward_change_std": 0.1709576854482293, |
| "reward_std": 0.6414528302848339, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": -0.076968085631961, |
| "step": 280 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3559.000030517578, |
| "epoch": 0.3211428571428571, |
| "grad_norm": 0.0494488850235939, |
| "kl": 3.0197203159332275e-05, |
| "lambda_div_used": 0.5301951244473457, |
| "learning_rate": 5.342952264838747e-07, |
| "loss": -0.0035, |
| "reward": -0.5142169296741486, |
| "reward_after_mean": -0.5142169296741486, |
| "reward_after_std": 0.22926145792007446, |
| "reward_before_mean": -0.2300790660083294, |
| "reward_before_std": 0.1324998252093792, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2841378580778837, |
| "reward_change_min": -0.4100039228796959, |
| "reward_change_std": 0.14742697216570377, |
| "reward_std": 0.22926146537065506, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2300790660083294, |
| "step": 281 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2757.7708587646484, |
| "epoch": 0.3222857142857143, |
| "grad_norm": 0.07136296480894089, |
| "kl": 1.5236437320709229e-05, |
| "lambda_div_used": 0.6041858941316605, |
| "learning_rate": 5.311559558218603e-07, |
| "loss": 0.0304, |
| "reward": 0.016035709530115128, |
| "reward_after_mean": 0.016035709530115128, |
| "reward_after_std": 0.5573885068297386, |
| "reward_before_mean": 0.4244079850614071, |
| "reward_before_std": 0.48719789227470756, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.40837226063013077, |
| "reward_change_min": -0.630510251969099, |
| "reward_change_std": 0.253003865480423, |
| "reward_std": 0.5573885291814804, |
| "rewards/accuracy_reward": 0.3541666716337204, |
| "rewards/cosine_scaled_reward": 0.07024132460355759, |
| "step": 282 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2882.5416946411133, |
| "epoch": 0.32342857142857145, |
| "grad_norm": 0.05982055515050888, |
| "kl": 1.1865049600601196e-05, |
| "lambda_div_used": 0.6191478446125984, |
| "learning_rate": 5.28017603591974e-07, |
| "loss": -0.015, |
| "reward": 0.10000502690672874, |
| "reward_after_mean": 0.10000502690672874, |
| "reward_after_std": 0.6287181153893471, |
| "reward_before_mean": 0.5279214177280664, |
| "reward_before_std": 0.5531216450035572, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4279164057224989, |
| "reward_change_min": -0.6583010666072369, |
| "reward_change_std": 0.2602456407621503, |
| "reward_std": 0.6287181228399277, |
| "rewards/accuracy_reward": 0.37500000931322575, |
| "rewards/cosine_scaled_reward": 0.1529214084148407, |
| "step": 283 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2187.083381652832, |
| "epoch": 0.32457142857142857, |
| "grad_norm": 0.20831985771656036, |
| "kl": 5.378853529691696e-05, |
| "lambda_div_used": 0.617803268134594, |
| "learning_rate": 5.248803227530763e-07, |
| "loss": 0.0301, |
| "reward": -0.09672576747834682, |
| "reward_after_mean": -0.09672576747834682, |
| "reward_after_std": 0.5820088647305965, |
| "reward_before_mean": 0.21344758570194244, |
| "reward_before_std": 0.544846129603684, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3101733736693859, |
| "reward_change_min": -0.5270103476941586, |
| "reward_change_std": 0.20029443874955177, |
| "reward_std": 0.5820088759064674, |
| "rewards/accuracy_reward": 0.22916667349636555, |
| "rewards/cosine_scaled_reward": -0.0157190952450037, |
| "step": 284 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2571.4583435058594, |
| "epoch": 0.32571428571428573, |
| "grad_norm": 0.05750874802470207, |
| "kl": 2.5823712348937988e-05, |
| "lambda_div_used": 0.5412982106208801, |
| "learning_rate": 5.21744266211809e-07, |
| "loss": 0.0429, |
| "reward": -0.3610886335372925, |
| "reward_after_mean": -0.3610886335372925, |
| "reward_after_std": 0.2963402010500431, |
| "reward_before_mean": 0.004524916410446167, |
| "reward_before_std": 0.18141429405659437, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36561354249715805, |
| "reward_change_min": -0.5212092585861683, |
| "reward_change_std": 0.19485185854136944, |
| "reward_std": 0.29634021408855915, |
| "rewards/accuracy_reward": 0.125, |
| "rewards/cosine_scaled_reward": -0.12047509290277958, |
| "step": 285 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2607.625030517578, |
| "epoch": 0.32685714285714285, |
| "grad_norm": 0.06422320753335953, |
| "kl": 2.036895602941513e-05, |
| "lambda_div_used": 0.593349277973175, |
| "learning_rate": 5.186095868151436e-07, |
| "loss": 0.0069, |
| "reward": -0.1434864792972803, |
| "reward_after_mean": -0.1434864792972803, |
| "reward_after_std": 0.4716298431158066, |
| "reward_before_mean": 0.18680993653833866, |
| "reward_before_std": 0.4280705275014043, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3302964009344578, |
| "reward_change_min": -0.5444469898939133, |
| "reward_change_std": 0.20489494875073433, |
| "reward_std": 0.4716298636049032, |
| "rewards/accuracy_reward": 0.2291666753590107, |
| "rewards/cosine_scaled_reward": -0.04235674999654293, |
| "step": 286 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2068.4375343322754, |
| "epoch": 0.328, |
| "grad_norm": 0.11217836290597916, |
| "kl": 3.282725811004639e-05, |
| "lambda_div_used": 0.576636016368866, |
| "learning_rate": 5.154764373429315e-07, |
| "loss": -0.0127, |
| "reward": -0.015125550329685211, |
| "reward_after_mean": -0.015125550329685211, |
| "reward_after_std": 0.44022079929709435, |
| "reward_before_mean": 0.4338516741991043, |
| "reward_before_std": 0.349846001714468, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4489772208034992, |
| "reward_change_min": -0.6400703266263008, |
| "reward_change_std": 0.25500839948654175, |
| "reward_std": 0.44022080302238464, |
| "rewards/accuracy_reward": 0.3125000074505806, |
| "rewards/cosine_scaled_reward": 0.12135165929794312, |
| "step": 287 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3021.1666870117188, |
| "epoch": 0.3291428571428571, |
| "grad_norm": 0.05326732248067856, |
| "kl": 2.1582163753919303e-05, |
| "lambda_div_used": 0.5814583152532578, |
| "learning_rate": 5.123449705004581e-07, |
| "loss": 0.0215, |
| "reward": -0.23482287488877773, |
| "reward_after_mean": -0.23482287488877773, |
| "reward_after_std": 0.4797380156815052, |
| "reward_before_mean": 0.10618079453706741, |
| "reward_before_std": 0.37177785113453865, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34100368432700634, |
| "reward_change_min": -0.5246531553566456, |
| "reward_change_std": 0.190623770467937, |
| "reward_std": 0.47973802499473095, |
| "rewards/accuracy_reward": 0.18750000186264515, |
| "rewards/cosine_scaled_reward": -0.08131920825690031, |
| "step": 288 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2210.0208435058594, |
| "epoch": 0.3302857142857143, |
| "grad_norm": 0.0918356254696846, |
| "kl": 3.3445656299591064e-05, |
| "lambda_div_used": 0.5579611882567406, |
| "learning_rate": 5.09215338910999e-07, |
| "loss": 0.0308, |
| "reward": -0.1890019178390503, |
| "reward_after_mean": -0.1890019178390503, |
| "reward_after_std": 0.38851393200457096, |
| "reward_before_mean": 0.2269407268613577, |
| "reward_before_std": 0.2580757178366184, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4159426633268595, |
| "reward_change_min": -0.5701811872422695, |
| "reward_change_std": 0.21641669981181622, |
| "reward_std": 0.38851393945515156, |
| "rewards/accuracy_reward": 0.27083333395421505, |
| "rewards/cosine_scaled_reward": -0.04389259871095419, |
| "step": 289 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1820.6666870117188, |
| "epoch": 0.3314285714285714, |
| "grad_norm": 0.10334110260009766, |
| "kl": 1.7639249563217163e-05, |
| "lambda_div_used": 0.649879202246666, |
| "learning_rate": 5.060876951083828e-07, |
| "loss": -0.0389, |
| "reward": 0.04819735325872898, |
| "reward_after_mean": 0.04819735325872898, |
| "reward_after_std": 0.752808591350913, |
| "reward_before_mean": 0.39133079699240625, |
| "reward_before_std": 0.7000208692625165, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3431334514170885, |
| "reward_change_min": -0.5988858677446842, |
| "reward_change_std": 0.2255254928022623, |
| "reward_std": 0.7528086155653, |
| "rewards/accuracy_reward": 0.3333333358168602, |
| "rewards/cosine_scaled_reward": 0.05799746699631214, |
| "step": 290 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2717.166732788086, |
| "epoch": 0.3325714285714286, |
| "grad_norm": 0.07149659842252731, |
| "kl": 9.842216968536377e-06, |
| "lambda_div_used": 0.6230586618185043, |
| "learning_rate": 5.02962191529556e-07, |
| "loss": 0.0634, |
| "reward": 0.04544975981116295, |
| "reward_after_mean": 0.04544975981116295, |
| "reward_after_std": 0.6337935384362936, |
| "reward_before_mean": 0.4299341347068548, |
| "reward_before_std": 0.5707205794751644, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.38448438234627247, |
| "reward_change_min": -0.6367709413170815, |
| "reward_change_std": 0.2398422248661518, |
| "reward_std": 0.6337935607880354, |
| "rewards/accuracy_reward": 0.33333333767950535, |
| "rewards/cosine_scaled_reward": 0.09660080214962363, |
| "step": 291 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3176.979202270508, |
| "epoch": 0.33371428571428574, |
| "grad_norm": 0.05150657147169113, |
| "kl": 2.0240433514118195e-05, |
| "lambda_div_used": 0.5560602247714996, |
| "learning_rate": 4.998389805071536e-07, |
| "loss": 0.062, |
| "reward": -0.4294139966368675, |
| "reward_after_mean": -0.4294139966368675, |
| "reward_after_std": 0.3452510107308626, |
| "reward_before_mean": -0.15922172274440527, |
| "reward_before_std": 0.2578093442134559, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27019229158759117, |
| "reward_change_min": -0.42562897875905037, |
| "reward_change_std": 0.15438843425363302, |
| "reward_std": 0.3452510181814432, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.20088838692754507, |
| "step": 292 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2630.9791946411133, |
| "epoch": 0.33485714285714285, |
| "grad_norm": 0.05954331159591675, |
| "kl": 1.2833625078201294e-05, |
| "lambda_div_used": 0.5555919855833054, |
| "learning_rate": 4.967182142620745e-07, |
| "loss": 0.0009, |
| "reward": -0.19852645695209503, |
| "reward_after_mean": -0.19852645695209503, |
| "reward_after_std": 0.3515181578695774, |
| "reward_before_mean": 0.20652466267347336, |
| "reward_before_std": 0.24862384609878063, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.40505112148821354, |
| "reward_change_min": -0.5747470110654831, |
| "reward_change_std": 0.22049889154732227, |
| "reward_std": 0.35151816345751286, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/cosine_scaled_reward": -0.022642023861408234, |
| "step": 293 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3005.5, |
| "epoch": 0.336, |
| "grad_norm": 0.06993379443883896, |
| "kl": 2.8274953365325928e-05, |
| "lambda_div_used": 0.5608418732881546, |
| "learning_rate": 4.93600044896063e-07, |
| "loss": -0.0248, |
| "reward": -0.3667301833629608, |
| "reward_after_mean": -0.3667301833629608, |
| "reward_after_std": 0.32350156269967556, |
| "reward_before_mean": -0.07280110754072666, |
| "reward_before_std": 0.2737845163792372, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29392906464636326, |
| "reward_change_min": -0.4593726359307766, |
| "reward_change_std": 0.17168805841356516, |
| "reward_std": 0.3235015720129013, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.15613443590700626, |
| "step": 294 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3221.2291717529297, |
| "epoch": 0.33714285714285713, |
| "grad_norm": 0.06364311277866364, |
| "kl": 1.817569136619568e-05, |
| "lambda_div_used": 0.5979067236185074, |
| "learning_rate": 4.904846243842949e-07, |
| "loss": 0.0107, |
| "reward": -0.259306401014328, |
| "reward_after_mean": -0.259306401014328, |
| "reward_after_std": 0.5024331342428923, |
| "reward_before_mean": 0.018260781886056066, |
| "reward_before_std": 0.45087322127074003, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.277567183598876, |
| "reward_change_min": -0.41658853366971016, |
| "reward_change_std": 0.1643626783043146, |
| "reward_std": 0.5024331398308277, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.10673921927809715, |
| "step": 295 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3282.8750610351562, |
| "epoch": 0.3382857142857143, |
| "grad_norm": 0.0698896273970604, |
| "kl": 2.3877248167991638e-05, |
| "lambda_div_used": 0.5625706240534782, |
| "learning_rate": 4.873721045679706e-07, |
| "loss": 0.0079, |
| "reward": -0.39721857756376266, |
| "reward_after_mean": -0.39721857756376266, |
| "reward_after_std": 0.34284412302076817, |
| "reward_before_mean": -0.1091517936438322, |
| "reward_before_std": 0.28544116113334894, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28806679882109165, |
| "reward_change_min": -0.4639837518334389, |
| "reward_change_std": 0.1706175785511732, |
| "reward_std": 0.3428441286087036, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.15081845596432686, |
| "step": 296 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3569.8541870117188, |
| "epoch": 0.3394285714285714, |
| "grad_norm": 0.04541197791695595, |
| "kl": 2.245139330625534e-05, |
| "lambda_div_used": 0.5649159774184227, |
| "learning_rate": 4.842626371469149e-07, |
| "loss": 0.0071, |
| "reward": -0.43140796944499016, |
| "reward_after_mean": -0.43140796944499016, |
| "reward_after_std": 0.3433863054960966, |
| "reward_before_mean": -0.17089181207120419, |
| "reward_before_std": 0.29400468710809946, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26051616482436657, |
| "reward_change_min": -0.4413977116346359, |
| "reward_change_std": 0.15714262332767248, |
| "reward_std": 0.3433863129466772, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2125584715977311, |
| "step": 297 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2834.479202270508, |
| "epoch": 0.3405714285714286, |
| "grad_norm": 0.07559877634048462, |
| "kl": 1.9371509552001953e-05, |
| "lambda_div_used": 0.5950812250375748, |
| "learning_rate": 4.811563736721829e-07, |
| "loss": -0.0217, |
| "reward": -0.13096300419420004, |
| "reward_after_mean": -0.13096300419420004, |
| "reward_after_std": 0.505608232691884, |
| "reward_before_mean": 0.22305661533027887, |
| "reward_before_std": 0.43375879526138306, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.354019645601511, |
| "reward_change_min": -0.5659407489001751, |
| "reward_change_std": 0.20670694950968027, |
| "reward_std": 0.5056082457304001, |
| "rewards/accuracy_reward": 0.29166667349636555, |
| "rewards/cosine_scaled_reward": -0.0686100497841835, |
| "step": 298 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3288.5833587646484, |
| "epoch": 0.3417142857142857, |
| "grad_norm": 0.05182049795985222, |
| "kl": 3.013480454683304e-05, |
| "lambda_div_used": 0.5971293970942497, |
| "learning_rate": 4.780534655386743e-07, |
| "loss": -0.0035, |
| "reward": -0.1649590004235506, |
| "reward_after_mean": -0.1649590004235506, |
| "reward_after_std": 0.476910138502717, |
| "reward_before_mean": 0.15552489459514618, |
| "reward_before_std": 0.44229450821876526, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3204838838428259, |
| "reward_change_min": -0.5031619034707546, |
| "reward_change_std": 0.19439208041876554, |
| "reward_std": 0.47691015154123306, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": -0.052808452397584915, |
| "step": 299 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3455.3958435058594, |
| "epoch": 0.34285714285714286, |
| "grad_norm": 0.059302717447280884, |
| "kl": 3.039836883544922e-05, |
| "lambda_div_used": 0.5659952610731125, |
| "learning_rate": 4.749540639777539e-07, |
| "loss": 0.0631, |
| "reward": -0.4138263203203678, |
| "reward_after_mean": -0.4138263203203678, |
| "reward_after_std": 0.35950295627117157, |
| "reward_before_mean": -0.14833886176347733, |
| "reward_before_std": 0.30170741491019726, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26548744924366474, |
| "reward_change_min": -0.45029690116643906, |
| "reward_change_std": 0.16043098457157612, |
| "reward_std": 0.35950295627117157, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.19000552780926228, |
| "step": 300 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2581.2917404174805, |
| "epoch": 0.344, |
| "grad_norm": 0.0829557478427887, |
| "kl": 1.7702579498291016e-05, |
| "lambda_div_used": 0.6020706444978714, |
| "learning_rate": 4.7185832004988133e-07, |
| "loss": 0.0033, |
| "reward": -0.2905968725681305, |
| "reward_after_mean": -0.2905968725681305, |
| "reward_after_std": 0.522302333265543, |
| "reward_before_mean": -0.03232934419065714, |
| "reward_before_std": 0.47291796933859587, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2582675274461508, |
| "reward_change_min": -0.4681916609406471, |
| "reward_change_std": 0.16694430727511644, |
| "reward_std": 0.5223023407161236, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.13649601582437754, |
| "step": 301 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2299.250026702881, |
| "epoch": 0.34514285714285714, |
| "grad_norm": 0.09259702265262604, |
| "kl": 2.473965287208557e-05, |
| "lambda_div_used": 0.6064160838723183, |
| "learning_rate": 4.68766384637248e-07, |
| "loss": 0.0049, |
| "reward": 0.1023632986471057, |
| "reward_after_mean": 0.1023632986471057, |
| "reward_after_std": 0.5881610047072172, |
| "reward_before_mean": 0.5643375236541033, |
| "reward_before_std": 0.48729276517406106, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.46197423338890076, |
| "reward_change_min": -0.6793596595525742, |
| "reward_change_std": 0.2643775464966893, |
| "reward_std": 0.588161014020443, |
| "rewards/accuracy_reward": 0.39583334513008595, |
| "rewards/cosine_scaled_reward": 0.16850417526438832, |
| "step": 302 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2496.604202270508, |
| "epoch": 0.3462857142857143, |
| "grad_norm": 0.10342813283205032, |
| "kl": 2.822279930114746e-05, |
| "lambda_div_used": 0.6173663139343262, |
| "learning_rate": 4.656784084364238e-07, |
| "loss": -0.0508, |
| "reward": -0.21569269057363272, |
| "reward_after_mean": -0.21569269057363272, |
| "reward_after_std": 0.5887450613081455, |
| "reward_before_mean": 0.048040480352938175, |
| "reward_before_std": 0.5433634500950575, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2637331634759903, |
| "reward_change_min": -0.4554433934390545, |
| "reward_change_std": 0.17205023765563965, |
| "reward_std": 0.5887450724840164, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.09779285965487361, |
| "step": 303 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2828.375030517578, |
| "epoch": 0.3474285714285714, |
| "grad_norm": 0.06273287534713745, |
| "kl": 3.515370190143585e-05, |
| "lambda_div_used": 0.5850896239280701, |
| "learning_rate": 4.6259454195101267e-07, |
| "loss": 0.0395, |
| "reward": -0.351226020604372, |
| "reward_after_mean": -0.351226020604372, |
| "reward_after_std": 0.4407376032322645, |
| "reward_before_mean": -0.091004628688097, |
| "reward_before_std": 0.39247662480920553, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2602214030921459, |
| "reward_change_min": -0.4692201763391495, |
| "reward_change_std": 0.16711712814867496, |
| "reward_std": 0.4407376106828451, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.1743379645049572, |
| "step": 304 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2963.270835876465, |
| "epoch": 0.3485714285714286, |
| "grad_norm": 0.07068406045436859, |
| "kl": 3.521144390106201e-05, |
| "lambda_div_used": 0.5555498078465462, |
| "learning_rate": 4.59514935484316e-07, |
| "loss": 0.0077, |
| "reward": -0.44899775832891464, |
| "reward_after_mean": -0.44899775832891464, |
| "reward_after_std": 0.32930242642760277, |
| "reward_before_mean": -0.17746215965598822, |
| "reward_before_std": 0.2488851365633309, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27153559774160385, |
| "reward_change_min": -0.41173194721341133, |
| "reward_change_std": 0.14773181919008493, |
| "reward_std": 0.32930243387818336, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.19829548941925168, |
| "step": 305 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2542.8750228881836, |
| "epoch": 0.3497142857142857, |
| "grad_norm": 0.09160702675580978, |
| "kl": 2.777576446533203e-05, |
| "lambda_div_used": 0.5710241869091988, |
| "learning_rate": 4.5643973913200837e-07, |
| "loss": 0.001, |
| "reward": -0.28438286297023296, |
| "reward_after_mean": -0.28438286297023296, |
| "reward_after_std": 0.45469350554049015, |
| "reward_before_mean": 0.05048683221684769, |
| "reward_before_std": 0.3197702756151557, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.33486970886588097, |
| "reward_change_min": -0.46306542679667473, |
| "reward_change_std": 0.1733616916462779, |
| "reward_std": 0.45469350926578045, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.11617984622716904, |
| "step": 306 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2358.9167098999023, |
| "epoch": 0.35085714285714287, |
| "grad_norm": 0.08308543264865875, |
| "kl": 2.5270506739616394e-05, |
| "lambda_div_used": 0.6283661872148514, |
| "learning_rate": 4.5336910277482155e-07, |
| "loss": 0.0233, |
| "reward": -0.023870151489973068, |
| "reward_after_mean": -0.023870151489973068, |
| "reward_after_std": 0.5968872811645269, |
| "reward_before_mean": 0.2986760139465332, |
| "reward_before_std": 0.602552474476397, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3225461672991514, |
| "reward_change_min": -0.5788566246628761, |
| "reward_change_std": 0.22817020770162344, |
| "reward_std": 0.5968873165547848, |
| "rewards/accuracy_reward": 0.2916666716337204, |
| "rewards/cosine_scaled_reward": 0.007009351626038551, |
| "step": 307 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3364.187530517578, |
| "epoch": 0.352, |
| "grad_norm": 0.04729427769780159, |
| "kl": 1.2226402759552002e-05, |
| "lambda_div_used": 0.5575956255197525, |
| "learning_rate": 4.503031760712397e-07, |
| "loss": -0.0147, |
| "reward": -0.364587739109993, |
| "reward_after_mean": -0.364587739109993, |
| "reward_after_std": 0.3172164801508188, |
| "reward_before_mean": -0.06630371138453484, |
| "reward_before_std": 0.25998193118721247, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29828402772545815, |
| "reward_change_min": -0.4606154337525368, |
| "reward_change_std": 0.17100436985492706, |
| "reward_std": 0.31721648946404457, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.14963705092668533, |
| "step": 308 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3246.750030517578, |
| "epoch": 0.35314285714285715, |
| "grad_norm": 0.054204076528549194, |
| "kl": 1.4858320355415344e-05, |
| "lambda_div_used": 0.6364770829677582, |
| "learning_rate": 4.4724210845020494e-07, |
| "loss": 0.0936, |
| "reward": -0.09930397570133209, |
| "reward_after_mean": -0.09930397570133209, |
| "reward_after_std": 0.6563027147203684, |
| "reward_before_mean": 0.18065341375768185, |
| "reward_before_std": 0.6305609010159969, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27995740808546543, |
| "reward_change_min": -0.5127242244780064, |
| "reward_change_std": 0.19019456766545773, |
| "reward_std": 0.6563027296215296, |
| "rewards/accuracy_reward": 0.2083333358168602, |
| "rewards/cosine_scaled_reward": -0.027679930441081524, |
| "step": 309 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2454.750015258789, |
| "epoch": 0.35428571428571426, |
| "grad_norm": 0.10188445448875427, |
| "kl": 3.9263395592570305e-05, |
| "lambda_div_used": 0.5863905549049377, |
| "learning_rate": 4.441860491038345e-07, |
| "loss": -0.0091, |
| "reward": -0.358197920024395, |
| "reward_after_mean": -0.358197920024395, |
| "reward_after_std": 0.4752412661910057, |
| "reward_before_mean": -0.10783447185531259, |
| "reward_before_std": 0.39106855262070894, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25036344304680824, |
| "reward_change_min": -0.36833222955465317, |
| "reward_change_std": 0.13517011515796185, |
| "reward_std": 0.47524126805365086, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.17033447965513915, |
| "step": 310 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2549.5000228881836, |
| "epoch": 0.3554285714285714, |
| "grad_norm": 0.07070475816726685, |
| "kl": 1.1476688086986542e-05, |
| "lambda_div_used": 0.6028245091438293, |
| "learning_rate": 4.4113514698014953e-07, |
| "loss": -0.0234, |
| "reward": -0.07257996033877134, |
| "reward_after_mean": -0.07257996033877134, |
| "reward_after_std": 0.5427698846906424, |
| "reward_before_mean": 0.32260693423449993, |
| "reward_before_std": 0.4735551681369543, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3951868824660778, |
| "reward_change_min": -0.6274620294570923, |
| "reward_change_std": 0.2447280865162611, |
| "reward_std": 0.5427699014544487, |
| "rewards/accuracy_reward": 0.3125000037252903, |
| "rewards/cosine_scaled_reward": 0.010106915608048439, |
| "step": 311 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2062.020851135254, |
| "epoch": 0.3565714285714286, |
| "grad_norm": 0.0864594504237175, |
| "kl": 2.2163614630699158e-05, |
| "lambda_div_used": 0.5457132831215858, |
| "learning_rate": 4.3808955077581546e-07, |
| "loss": 0.0122, |
| "reward": -0.03406492620706558, |
| "reward_after_mean": -0.03406492620706558, |
| "reward_after_std": 0.3851391561329365, |
| "reward_before_mean": 0.5170513242483139, |
| "reward_before_std": 0.20092127658426762, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5511162541806698, |
| "reward_change_min": -0.7572819888591766, |
| "reward_change_std": 0.2855915669351816, |
| "reward_std": 0.38513917103409767, |
| "rewards/accuracy_reward": 0.375, |
| "rewards/cosine_scaled_reward": 0.1420513167977333, |
| "step": 312 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2939.958351135254, |
| "epoch": 0.3577142857142857, |
| "grad_norm": 0.06731264293193817, |
| "kl": 2.4283304810523987e-05, |
| "lambda_div_used": 0.5877447426319122, |
| "learning_rate": 4.350494089288943e-07, |
| "loss": -0.0439, |
| "reward": -0.15467195864766836, |
| "reward_after_mean": -0.15467195864766836, |
| "reward_after_std": 0.522945849224925, |
| "reward_before_mean": 0.20540180057287216, |
| "reward_before_std": 0.40051606576889753, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3600737862288952, |
| "reward_change_min": -0.5124829597771168, |
| "reward_change_std": 0.19857864920049906, |
| "reward_std": 0.5229458846151829, |
| "rewards/accuracy_reward": 0.2291666679084301, |
| "rewards/cosine_scaled_reward": -0.02376485476270318, |
| "step": 313 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2541.18754196167, |
| "epoch": 0.3588571428571429, |
| "grad_norm": 0.08012691885232925, |
| "kl": 3.474205732345581e-05, |
| "lambda_div_used": 0.607881672680378, |
| "learning_rate": 4.3201486961161093e-07, |
| "loss": 0.0168, |
| "reward": -0.20861644856631756, |
| "reward_after_mean": -0.20861644856631756, |
| "reward_after_std": 0.5089061111211777, |
| "reward_before_mean": 0.08002243563532829, |
| "reward_before_std": 0.49900877848267555, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28863888792693615, |
| "reward_change_min": -0.548176895827055, |
| "reward_change_std": 0.2009973768144846, |
| "reward_std": 0.5089061167091131, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.08664422971196473, |
| "step": 314 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3124.3333740234375, |
| "epoch": 0.36, |
| "grad_norm": 0.0558871328830719, |
| "kl": 1.9735191017389297e-05, |
| "lambda_div_used": 0.5734824016690254, |
| "learning_rate": 4.2898608072313045e-07, |
| "loss": 0.0494, |
| "reward": -0.1729298224672675, |
| "reward_after_mean": -0.1729298224672675, |
| "reward_after_std": 0.45045475475490093, |
| "reward_before_mean": 0.20758753083646297, |
| "reward_before_std": 0.33522335812449455, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3805173486471176, |
| "reward_change_min": -0.5440769977867603, |
| "reward_change_std": 0.20876472163945436, |
| "reward_std": 0.4504547640681267, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": -0.02157914894632995, |
| "step": 315 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3509.5416870117188, |
| "epoch": 0.36114285714285715, |
| "grad_norm": 0.05331319198012352, |
| "kl": 2.981536090373993e-05, |
| "lambda_div_used": 0.5893311724066734, |
| "learning_rate": 4.2596318988235037e-07, |
| "loss": 0.0135, |
| "reward": -0.22396450489759445, |
| "reward_after_mean": -0.22396450489759445, |
| "reward_after_std": 0.43868801929056644, |
| "reward_before_mean": 0.07456053979694843, |
| "reward_before_std": 0.414919788017869, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29852502048015594, |
| "reward_change_min": -0.4823254942893982, |
| "reward_change_std": 0.1912739286199212, |
| "reward_std": 0.43868803791701794, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.09210613742470741, |
| "step": 316 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3099.5833435058594, |
| "epoch": 0.36228571428571427, |
| "grad_norm": 0.06200157478451729, |
| "kl": 3.091990947723389e-05, |
| "lambda_div_used": 0.5825675800442696, |
| "learning_rate": 4.2294634442070553e-07, |
| "loss": 0.0437, |
| "reward": -0.35854507237672806, |
| "reward_after_mean": -0.35854507237672806, |
| "reward_after_std": 0.43180895783007145, |
| "reward_before_mean": -0.102005859836936, |
| "reward_before_std": 0.3769839182496071, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25653921999037266, |
| "reward_change_min": -0.4170425795018673, |
| "reward_change_std": 0.15367608424276114, |
| "reward_std": 0.4318089634180069, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.18533920124173164, |
| "step": 317 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2281.000068664551, |
| "epoch": 0.36342857142857143, |
| "grad_norm": 0.0932127833366394, |
| "kl": 3.602728247642517e-05, |
| "lambda_div_used": 0.6223095878958702, |
| "learning_rate": 4.1993569137498776e-07, |
| "loss": 0.0631, |
| "reward": -0.07363937655463815, |
| "reward_after_mean": -0.07363937655463815, |
| "reward_after_std": 0.5691333152353764, |
| "reward_before_mean": 0.24711408838629723, |
| "reward_before_std": 0.5656038168817759, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3207534924149513, |
| "reward_change_min": -0.5814453661441803, |
| "reward_change_std": 0.2219030074775219, |
| "reward_std": 0.569133322685957, |
| "rewards/accuracy_reward": 0.2500000074505806, |
| "rewards/cosine_scaled_reward": -0.0028859074227511883, |
| "step": 318 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2854.5625076293945, |
| "epoch": 0.36457142857142855, |
| "grad_norm": 0.08494079858064651, |
| "kl": 2.7507543563842773e-05, |
| "lambda_div_used": 0.5380512997508049, |
| "learning_rate": 4.1693137748017915e-07, |
| "loss": -0.0389, |
| "reward": -0.4965253435075283, |
| "reward_after_mean": -0.4965253435075283, |
| "reward_after_std": 0.2333353590220213, |
| "reward_before_mean": -0.2120634987950325, |
| "reward_before_std": 0.16711975168436766, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28446184657514095, |
| "reward_change_min": -0.4124374948441982, |
| "reward_change_std": 0.15511877462267876, |
| "reward_std": 0.2333353627473116, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2120634950697422, |
| "step": 319 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1964.8958740234375, |
| "epoch": 0.3657142857142857, |
| "grad_norm": 0.09068436175584793, |
| "kl": 3.33394855260849e-05, |
| "lambda_div_used": 0.6160280704498291, |
| "learning_rate": 4.1393354916230005e-07, |
| "loss": 0.0117, |
| "reward": -0.19740011962130666, |
| "reward_after_mean": -0.19740011962130666, |
| "reward_after_std": 0.5963947810232639, |
| "reward_before_mean": 0.0693174353800714, |
| "reward_before_std": 0.5336382519453764, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26671755872666836, |
| "reward_change_min": -0.4143567681312561, |
| "reward_change_std": 0.15817437414079905, |
| "reward_std": 0.5963947977870703, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.05568256159313023, |
| "step": 320 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1974.8958778381348, |
| "epoch": 0.3668571428571429, |
| "grad_norm": 0.0693729817867279, |
| "kl": 1.574307680130005e-05, |
| "lambda_div_used": 0.6290735602378845, |
| "learning_rate": 4.1094235253127374e-07, |
| "loss": 0.0387, |
| "reward": 0.11023577488958836, |
| "reward_after_mean": 0.11023577488958836, |
| "reward_after_std": 0.6442763805389404, |
| "reward_before_mean": 0.5218545235693455, |
| "reward_before_std": 0.6013733670115471, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4116187300533056, |
| "reward_change_min": -0.6836260408163071, |
| "reward_change_std": 0.2705372450873256, |
| "reward_std": 0.6442763898521662, |
| "rewards/accuracy_reward": 0.3958333395421505, |
| "rewards/cosine_scaled_reward": 0.12602117005735636, |
| "step": 321 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2837.833381652832, |
| "epoch": 0.368, |
| "grad_norm": 0.10218925029039383, |
| "kl": 4.9501657485961914e-05, |
| "lambda_div_used": 0.6059064492583275, |
| "learning_rate": 4.079579333738039e-07, |
| "loss": -0.0178, |
| "reward": -0.2954628551378846, |
| "reward_after_mean": -0.2954628551378846, |
| "reward_after_std": 0.5665331184864044, |
| "reward_before_mean": -0.04816420469433069, |
| "reward_before_std": 0.48093545995652676, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24729863554239273, |
| "reward_change_min": -0.3516420107334852, |
| "reward_change_std": 0.13184416200965643, |
| "reward_std": 0.5665331222116947, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.13149754563346505, |
| "step": 322 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3015.666717529297, |
| "epoch": 0.36914285714285716, |
| "grad_norm": 0.0596209317445755, |
| "kl": 1.7877668142318726e-05, |
| "lambda_div_used": 0.5565166473388672, |
| "learning_rate": 4.0498043714627006e-07, |
| "loss": 0.0465, |
| "reward": -0.16177266091108322, |
| "reward_after_mean": -0.16177266091108322, |
| "reward_after_std": 0.42436218820512295, |
| "reward_before_mean": 0.2816486116498709, |
| "reward_before_std": 0.2578566027805209, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4434212874621153, |
| "reward_change_min": -0.6335049495100975, |
| "reward_change_std": 0.23841418512165546, |
| "reward_std": 0.4243622049689293, |
| "rewards/accuracy_reward": 0.2916666679084301, |
| "rewards/cosine_scaled_reward": -0.010018057189881802, |
| "step": 323 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2751.6458435058594, |
| "epoch": 0.3702857142857143, |
| "grad_norm": 0.10376841574907303, |
| "kl": 2.1306797862052917e-05, |
| "lambda_div_used": 0.5670785158872604, |
| "learning_rate": 4.020100089676376e-07, |
| "loss": -0.0038, |
| "reward": -0.10797288408502936, |
| "reward_after_mean": -0.10797288408502936, |
| "reward_after_std": 0.4300071895122528, |
| "reward_before_mean": 0.3295288155786693, |
| "reward_before_std": 0.3011532872915268, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.43750171549618244, |
| "reward_change_min": -0.6185614429414272, |
| "reward_change_std": 0.2333526872098446, |
| "reward_std": 0.43000719882547855, |
| "rewards/accuracy_reward": 0.25000000558793545, |
| "rewards/cosine_scaled_reward": 0.07952881557866931, |
| "step": 324 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2943.8958587646484, |
| "epoch": 0.37142857142857144, |
| "grad_norm": 0.07239633053541183, |
| "kl": 9.275972843170166e-06, |
| "lambda_div_used": 0.6356078162789345, |
| "learning_rate": 3.9904679361238526e-07, |
| "loss": 0.0667, |
| "reward": -0.050798285752534866, |
| "reward_after_mean": -0.050798285752534866, |
| "reward_after_std": 0.6440786644816399, |
| "reward_before_mean": 0.24087253957986832, |
| "reward_before_std": 0.6331603992730379, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2916708290576935, |
| "reward_change_min": -0.4932599440217018, |
| "reward_change_std": 0.20057004038244486, |
| "reward_std": 0.6440786886960268, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": -0.02996080555021763, |
| "step": 325 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2375.645851135254, |
| "epoch": 0.37257142857142855, |
| "grad_norm": 0.09299691766500473, |
| "kl": 2.9705464839935303e-05, |
| "lambda_div_used": 0.5330315083265305, |
| "learning_rate": 3.9609093550344907e-07, |
| "loss": -0.0979, |
| "reward": -0.20862240344285965, |
| "reward_after_mean": -0.20862240344285965, |
| "reward_after_std": 0.3296848703175783, |
| "reward_before_mean": 0.2517512815538794, |
| "reward_before_std": 0.1450797226279974, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.460373692214489, |
| "reward_change_min": -0.6264396719634533, |
| "reward_change_std": 0.23280893173068762, |
| "reward_std": 0.3296848740428686, |
| "rewards/accuracy_reward": 0.25, |
| "rewards/cosine_scaled_reward": 0.001751287141814828, |
| "step": 326 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2810.187511444092, |
| "epoch": 0.3737142857142857, |
| "grad_norm": 0.06210591271519661, |
| "kl": 8.400529623031616e-06, |
| "lambda_div_used": 0.5783005133271217, |
| "learning_rate": 3.931425787051832e-07, |
| "loss": -0.0483, |
| "reward": -0.11793380603194237, |
| "reward_after_mean": -0.11793380603194237, |
| "reward_after_std": 0.45353105291724205, |
| "reward_before_mean": 0.2935123089700937, |
| "reward_before_std": 0.3566841436550021, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.41144610941410065, |
| "reward_change_min": -0.6089312434196472, |
| "reward_change_std": 0.23675687983632088, |
| "reward_std": 0.4535310585051775, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": 0.022678975947201252, |
| "step": 327 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3518.8958435058594, |
| "epoch": 0.37485714285714283, |
| "grad_norm": 0.05192619562149048, |
| "kl": 2.485513687133789e-05, |
| "lambda_div_used": 0.5715365409851074, |
| "learning_rate": 3.902018669163384e-07, |
| "loss": 0.0155, |
| "reward": -0.4497411046177149, |
| "reward_after_mean": -0.4497411046177149, |
| "reward_after_std": 0.40177739411592484, |
| "reward_before_mean": -0.21567542850971222, |
| "reward_before_std": 0.32284149527549744, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.23406569100916386, |
| "reward_change_min": -0.33814629539847374, |
| "reward_change_std": 0.12489900179207325, |
| "reward_std": 0.40177739597857, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2573420889675617, |
| "step": 328 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2243.0416946411133, |
| "epoch": 0.376, |
| "grad_norm": 0.10950610786676407, |
| "kl": 2.469681203365326e-05, |
| "lambda_div_used": 0.6176783889532089, |
| "learning_rate": 3.872689434630585e-07, |
| "loss": -0.0562, |
| "reward": -0.013319691643118858, |
| "reward_after_mean": -0.013319691643118858, |
| "reward_after_std": 0.5674807205796242, |
| "reward_before_mean": 0.3346351385116577, |
| "reward_before_std": 0.5435493532568216, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34795483760535717, |
| "reward_change_min": -0.5437430925667286, |
| "reward_change_std": 0.22110824659466743, |
| "reward_std": 0.5674807410687208, |
| "rewards/accuracy_reward": 0.27083334140479565, |
| "rewards/cosine_scaled_reward": 0.06380179291591048, |
| "step": 329 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2141.479190826416, |
| "epoch": 0.37714285714285717, |
| "grad_norm": 0.10051169991493225, |
| "kl": 4.89354133605957e-05, |
| "lambda_div_used": 0.5594945177435875, |
| "learning_rate": 3.843439512918949e-07, |
| "loss": -0.0064, |
| "reward": -0.43674849811941385, |
| "reward_after_mean": -0.43674849811941385, |
| "reward_after_std": 0.32430145144462585, |
| "reward_before_mean": -0.17029727809131145, |
| "reward_before_std": 0.2681279256939888, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.266451220959425, |
| "reward_change_min": -0.44791119545698166, |
| "reward_change_std": 0.1575082140043378, |
| "reward_std": 0.32430145516991615, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2119639493757859, |
| "step": 330 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2564.604202270508, |
| "epoch": 0.3782857142857143, |
| "grad_norm": 0.12382425367832184, |
| "kl": 4.225596785545349e-05, |
| "lambda_div_used": 0.5574543103575706, |
| "learning_rate": 3.8142703296283953e-07, |
| "loss": 0.0413, |
| "reward": -0.41342686861753464, |
| "reward_after_mean": -0.41342686861753464, |
| "reward_after_std": 0.33145094849169254, |
| "reward_before_mean": -0.13134576752781868, |
| "reward_before_std": 0.25930391903966665, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2820810880511999, |
| "reward_change_min": -0.4496995583176613, |
| "reward_change_std": 0.16215670108795166, |
| "reward_std": 0.33145095966756344, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.17301243357360363, |
| "step": 331 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2420.6666870117188, |
| "epoch": 0.37942857142857145, |
| "grad_norm": 0.07938601076602936, |
| "kl": 2.1241605281829834e-05, |
| "lambda_div_used": 0.5416555106639862, |
| "learning_rate": 3.785183306423767e-07, |
| "loss": 0.0287, |
| "reward": -0.21761326864361763, |
| "reward_after_mean": -0.21761326864361763, |
| "reward_after_std": 0.34953613951802254, |
| "reward_before_mean": 0.2352729644626379, |
| "reward_before_std": 0.18298226408660412, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.45288625732064247, |
| "reward_change_min": -0.6215734221041203, |
| "reward_change_std": 0.2345103258267045, |
| "reward_std": 0.34953614324331284, |
| "rewards/accuracy_reward": 0.25, |
| "rewards/cosine_scaled_reward": -0.014727018773555756, |
| "step": 332 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2350.6875381469727, |
| "epoch": 0.38057142857142856, |
| "grad_norm": 0.08657004684209824, |
| "kl": 2.1725893020629883e-05, |
| "lambda_div_used": 0.6729468256235123, |
| "learning_rate": 3.7561798609655373e-07, |
| "loss": 0.0369, |
| "reward": -0.01155824027955532, |
| "reward_after_mean": -0.01155824027955532, |
| "reward_after_std": 0.8104779217392206, |
| "reward_before_mean": 0.23989354074001312, |
| "reward_before_std": 0.8117707390338182, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2514517717063427, |
| "reward_change_min": -0.5040576457977295, |
| "reward_change_std": 0.19348772894591093, |
| "reward_std": 0.810477938503027, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": -0.030939804390072823, |
| "step": 333 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3491.6041870117188, |
| "epoch": 0.38171428571428573, |
| "grad_norm": 0.05208956450223923, |
| "kl": 1.1973665095865726e-05, |
| "lambda_div_used": 0.5869207382202148, |
| "learning_rate": 3.72726140684072e-07, |
| "loss": -0.009, |
| "reward": -0.2944636158645153, |
| "reward_after_mean": -0.2944636158645153, |
| "reward_after_std": 0.43844909220933914, |
| "reward_before_mean": -0.013160821050405502, |
| "reward_before_std": 0.40495526185259223, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2813027612864971, |
| "reward_change_min": -0.4717176593840122, |
| "reward_change_std": 0.1835445323958993, |
| "reward_std": 0.4384490940719843, |
| "rewards/accuracy_reward": 0.125, |
| "rewards/cosine_scaled_reward": -0.13816083781421185, |
| "step": 334 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2589.0833892822266, |
| "epoch": 0.38285714285714284, |
| "grad_norm": 0.0683097094297409, |
| "kl": 1.9595026969909668e-05, |
| "lambda_div_used": 0.6027273386716843, |
| "learning_rate": 3.6984293534939737e-07, |
| "loss": -0.0488, |
| "reward": 0.005996011197566986, |
| "reward_after_mean": 0.005996011197566986, |
| "reward_after_std": 0.5422599408775568, |
| "reward_before_mean": 0.41999348998069763, |
| "reward_before_std": 0.4701922629028559, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.41399746760725975, |
| "reward_change_min": -0.623339332640171, |
| "reward_change_std": 0.24341293703764677, |
| "reward_std": 0.5422599650919437, |
| "rewards/accuracy_reward": 0.33333334140479565, |
| "rewards/cosine_scaled_reward": 0.08666014112532139, |
| "step": 335 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3162.1458740234375, |
| "epoch": 0.384, |
| "grad_norm": 0.067026786506176, |
| "kl": 2.0024715922772884e-05, |
| "lambda_div_used": 0.5950900241732597, |
| "learning_rate": 3.6696851061588994e-07, |
| "loss": 0.0138, |
| "reward": -0.16571878641843796, |
| "reward_after_mean": -0.16571878641843796, |
| "reward_after_std": 0.4802239239215851, |
| "reward_before_mean": 0.14471458829939365, |
| "reward_before_std": 0.4368621027097106, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.31043340265750885, |
| "reward_change_min": -0.45523249730467796, |
| "reward_change_std": 0.18632046319544315, |
| "reward_std": 0.48022393323481083, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": -0.06361875729635358, |
| "step": 336 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3228.916717529297, |
| "epoch": 0.3851428571428571, |
| "grad_norm": 0.05528295040130615, |
| "kl": 1.9825994968414307e-05, |
| "lambda_div_used": 0.6006625667214394, |
| "learning_rate": 3.641030065789562e-07, |
| "loss": 0.0552, |
| "reward": -0.2433483125641942, |
| "reward_after_mean": -0.2433483125641942, |
| "reward_after_std": 0.5013067033141851, |
| "reward_before_mean": 0.03825543075799942, |
| "reward_before_std": 0.4624515902251005, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28160375356674194, |
| "reward_change_min": -0.4847262054681778, |
| "reward_change_std": 0.17836903873831034, |
| "reward_std": 0.501306714490056, |
| "rewards/accuracy_reward": 0.14583333767950535, |
| "rewards/cosine_scaled_reward": -0.10757789574563503, |
| "step": 337 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2393.645866394043, |
| "epoch": 0.3862857142857143, |
| "grad_norm": 0.10292567312717438, |
| "kl": 3.5159289836883545e-05, |
| "lambda_div_used": 0.6184682846069336, |
| "learning_rate": 3.612465628992203e-07, |
| "loss": 0.1082, |
| "reward": -0.010080082342028618, |
| "reward_after_mean": -0.010080082342028618, |
| "reward_after_std": 0.6298389285802841, |
| "reward_before_mean": 0.3685674872249365, |
| "reward_before_std": 0.546540604904294, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3786476030945778, |
| "reward_change_min": -0.5825354494154453, |
| "reward_change_std": 0.2281006295233965, |
| "reward_std": 0.6298389509320259, |
| "rewards/accuracy_reward": 0.3125000037252903, |
| "rewards/cosine_scaled_reward": 0.05606749979779124, |
| "step": 338 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3026.9583892822266, |
| "epoch": 0.38742857142857146, |
| "grad_norm": 0.0688738226890564, |
| "kl": 2.4460256099700928e-05, |
| "lambda_div_used": 0.5621443763375282, |
| "learning_rate": 3.5839931879571725e-07, |
| "loss": -0.0178, |
| "reward": -0.32898029685020447, |
| "reward_after_mean": -0.32898029685020447, |
| "reward_after_std": 0.3227359801530838, |
| "reward_before_mean": -0.01589403674006462, |
| "reward_before_std": 0.2781273443251848, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.31308628246188164, |
| "reward_change_min": -0.48185204714536667, |
| "reward_change_std": 0.1813461184501648, |
| "reward_std": 0.3227359913289547, |
| "rewards/accuracy_reward": 0.1041666716337204, |
| "rewards/cosine_scaled_reward": -0.12006070092320442, |
| "step": 339 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2447.937515258789, |
| "epoch": 0.38857142857142857, |
| "grad_norm": 0.10290948301553726, |
| "kl": 2.363821113249287e-05, |
| "lambda_div_used": 0.5592405423521996, |
| "learning_rate": 3.555614130391079e-07, |
| "loss": -0.0046, |
| "reward": -0.42997122183442116, |
| "reward_after_mean": -0.42997122183442116, |
| "reward_after_std": 0.33643546886742115, |
| "reward_before_mean": -0.1512418081983924, |
| "reward_before_std": 0.2663228642195463, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2787294201552868, |
| "reward_change_min": -0.4129678010940552, |
| "reward_change_std": 0.1523351836949587, |
| "reward_std": 0.3364354781806469, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.17207514215260744, |
| "step": 340 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2488.9583587646484, |
| "epoch": 0.38971428571428574, |
| "grad_norm": 0.061042170971632004, |
| "kl": 1.909211277961731e-05, |
| "lambda_div_used": 0.5834595933556557, |
| "learning_rate": 3.5273298394491515e-07, |
| "loss": -0.0023, |
| "reward": 0.009830114664509892, |
| "reward_after_mean": 0.009830114664509892, |
| "reward_after_std": 0.5199873205274343, |
| "reward_before_mean": 0.4715769328176975, |
| "reward_before_std": 0.38039534725248814, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.46174679696559906, |
| "reward_change_min": -0.6357490979135036, |
| "reward_change_std": 0.2529716519638896, |
| "reward_std": 0.5199873335659504, |
| "rewards/accuracy_reward": 0.33333334140479565, |
| "rewards/cosine_scaled_reward": 0.13824361143633723, |
| "step": 341 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3020.520854949951, |
| "epoch": 0.39085714285714285, |
| "grad_norm": 0.07325860857963562, |
| "kl": 2.4762004613876343e-05, |
| "lambda_div_used": 0.5985761880874634, |
| "learning_rate": 3.4991416936678276e-07, |
| "loss": 0.0007, |
| "reward": -0.06264204811304808, |
| "reward_after_mean": -0.06264204811304808, |
| "reward_after_std": 0.5387034490704536, |
| "reward_before_mean": 0.33080895524472, |
| "reward_before_std": 0.45346040464937687, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3934510052204132, |
| "reward_change_min": -0.5952773988246918, |
| "reward_change_std": 0.23232300952076912, |
| "reward_std": 0.5387034583836794, |
| "rewards/accuracy_reward": 0.29166666977107525, |
| "rewards/cosine_scaled_reward": 0.03914228640496731, |
| "step": 342 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3405.1875610351562, |
| "epoch": 0.392, |
| "grad_norm": 0.04665439575910568, |
| "kl": 2.7433037757873535e-05, |
| "lambda_div_used": 0.6150833070278168, |
| "learning_rate": 3.471051066897562e-07, |
| "loss": 0.0509, |
| "reward": -0.1582264108583331, |
| "reward_after_mean": -0.1582264108583331, |
| "reward_after_std": 0.5790915079414845, |
| "reward_before_mean": 0.13680532574653625, |
| "reward_before_std": 0.5264813583344221, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29503174126148224, |
| "reward_change_min": -0.4708465002477169, |
| "reward_change_std": 0.1767220702022314, |
| "reward_std": 0.5790915191173553, |
| "rewards/accuracy_reward": 0.1875000074505806, |
| "rewards/cosine_scaled_reward": -0.050694676116108894, |
| "step": 343 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2360.875045776367, |
| "epoch": 0.3931428571428571, |
| "grad_norm": 0.07271434366703033, |
| "kl": 1.041218638420105e-05, |
| "lambda_div_used": 0.5770246461033821, |
| "learning_rate": 3.4430593282358777e-07, |
| "loss": -0.0008, |
| "reward": 0.04454847797751427, |
| "reward_after_mean": 0.04454847797751427, |
| "reward_after_std": 0.5018230397254229, |
| "reward_before_mean": 0.5492894258350134, |
| "reward_before_std": 0.35145203582942486, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5047409404069185, |
| "reward_change_min": -0.7455123476684093, |
| "reward_change_std": 0.2790954224765301, |
| "reward_std": 0.5018230620771646, |
| "rewards/accuracy_reward": 0.4375, |
| "rewards/cosine_scaled_reward": 0.1117894072085619, |
| "step": 344 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3030.0833740234375, |
| "epoch": 0.3942857142857143, |
| "grad_norm": 0.0703606829047203, |
| "kl": 3.1925737857818604e-05, |
| "lambda_div_used": 0.5804503262042999, |
| "learning_rate": 3.4151678419606233e-07, |
| "loss": 0.0646, |
| "reward": -0.22463034093379974, |
| "reward_after_mean": -0.22463034093379974, |
| "reward_after_std": 0.41369511373341084, |
| "reward_before_mean": 0.09700941108167171, |
| "reward_before_std": 0.37086532823741436, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.32163975574076176, |
| "reward_change_min": -0.5076877251267433, |
| "reward_change_std": 0.1956562791019678, |
| "reward_std": 0.4136951379477978, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.06965726800262928, |
| "step": 345 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3186.6458740234375, |
| "epoch": 0.3954285714285714, |
| "grad_norm": 0.05292130261659622, |
| "kl": 1.4215707778930664e-05, |
| "lambda_div_used": 0.5958857163786888, |
| "learning_rate": 3.387377967463493e-07, |
| "loss": 0.0331, |
| "reward": -0.24454469978809357, |
| "reward_after_mean": -0.24454469978809357, |
| "reward_after_std": 0.48920249938964844, |
| "reward_before_mean": 0.04633236164227128, |
| "reward_before_std": 0.44041235372424126, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29087705351412296, |
| "reward_change_min": -0.4697858430445194, |
| "reward_change_std": 0.179568306542933, |
| "reward_std": 0.4892025087028742, |
| "rewards/accuracy_reward": 0.16666667349636555, |
| "rewards/cosine_scaled_reward": -0.12033431546296924, |
| "step": 346 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3055.4583587646484, |
| "epoch": 0.3965714285714286, |
| "grad_norm": 0.07354738563299179, |
| "kl": 6.0675665736198425e-06, |
| "lambda_div_used": 0.5582710355520248, |
| "learning_rate": 3.359691059183761e-07, |
| "loss": 0.0132, |
| "reward": -0.42621116526424885, |
| "reward_after_mean": -0.42621116526424885, |
| "reward_after_std": 0.3322943150997162, |
| "reward_before_mean": -0.15516536496579647, |
| "reward_before_std": 0.2633455842733383, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2710457965731621, |
| "reward_change_min": -0.4307188205420971, |
| "reward_change_std": 0.15532648842781782, |
| "reward_std": 0.33229432441294193, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.19683203659951687, |
| "step": 347 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2852.4583435058594, |
| "epoch": 0.3977142857142857, |
| "grad_norm": 0.09605983644723892, |
| "kl": 3.167241811752319e-05, |
| "lambda_div_used": 0.5788434967398643, |
| "learning_rate": 3.3321084665422803e-07, |
| "loss": -0.0017, |
| "reward": -0.2087371125817299, |
| "reward_after_mean": -0.2087371125817299, |
| "reward_after_std": 0.40125221759080887, |
| "reward_before_mean": 0.12491290923207998, |
| "reward_before_std": 0.36025606002658606, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3336500097066164, |
| "reward_change_min": -0.5146553069353104, |
| "reward_change_std": 0.20004253275692463, |
| "reward_std": 0.40125224366784096, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.04175376985222101, |
| "step": 348 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2964.875015258789, |
| "epoch": 0.39885714285714285, |
| "grad_norm": 0.06869405508041382, |
| "kl": 1.731887459754944e-05, |
| "lambda_div_used": 0.5606642514467239, |
| "learning_rate": 3.3046315338757026e-07, |
| "loss": 0.0031, |
| "reward": -0.2807905152440071, |
| "reward_after_mean": -0.2807905152440071, |
| "reward_after_std": 0.39816635474562645, |
| "reward_before_mean": 0.08202904835343361, |
| "reward_before_std": 0.2763519547879696, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36281956918537617, |
| "reward_change_min": -0.54488330706954, |
| "reward_change_std": 0.20150058157742023, |
| "reward_std": 0.39816636219620705, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.0846376121044159, |
| "step": 349 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2521.9791870117188, |
| "epoch": 0.4, |
| "grad_norm": 0.10303740203380585, |
| "kl": 2.6671215891838074e-05, |
| "lambda_div_used": 0.6121688261628151, |
| "learning_rate": 3.2772616003709616e-07, |
| "loss": -0.0711, |
| "reward": -0.04820730444043875, |
| "reward_after_mean": -0.04820730444043875, |
| "reward_after_std": 0.5604431573301554, |
| "reward_before_mean": 0.287683189380914, |
| "reward_before_std": 0.5214535812847316, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3358904607594013, |
| "reward_change_min": -0.5252225995063782, |
| "reward_change_std": 0.21241459622979164, |
| "reward_std": 0.5604431666433811, |
| "rewards/accuracy_reward": 0.27083334140479565, |
| "rewards/cosine_scaled_reward": 0.016849845182150602, |
| "step": 350 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3268.250030517578, |
| "epoch": 0.40114285714285713, |
| "grad_norm": 0.04833042621612549, |
| "kl": 2.4568289518356323e-05, |
| "lambda_div_used": 0.5601532310247421, |
| "learning_rate": 3.250000000000001e-07, |
| "loss": 0.0519, |
| "reward": -0.250777292996645, |
| "reward_after_mean": -0.250777292996645, |
| "reward_after_std": 0.38171103224158287, |
| "reward_before_mean": 0.1175131555646658, |
| "reward_before_std": 0.2718004435300827, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36829047463834286, |
| "reward_change_min": -0.5422449931502342, |
| "reward_change_std": 0.20415859669446945, |
| "reward_std": 0.3817110415548086, |
| "rewards/accuracy_reward": 0.1875, |
| "rewards/cosine_scaled_reward": -0.06998682580888271, |
| "step": 351 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2589.333366394043, |
| "epoch": 0.4022857142857143, |
| "grad_norm": 0.06552080065011978, |
| "kl": 1.905485987663269e-05, |
| "lambda_div_used": 0.6104472130537033, |
| "learning_rate": 3.222848061454764e-07, |
| "loss": 0.0025, |
| "reward": -0.09347447147592902, |
| "reward_after_mean": -0.09347447147592902, |
| "reward_after_std": 0.6156985405832529, |
| "reward_before_mean": 0.26158976616716245, |
| "reward_before_std": 0.5061299707740545, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3550642393529415, |
| "reward_change_min": -0.5232127867639065, |
| "reward_change_std": 0.1957317991182208, |
| "reward_std": 0.6156985703855753, |
| "rewards/accuracy_reward": 0.25000000186264515, |
| "rewards/cosine_scaled_reward": 0.01158975763246417, |
| "step": 352 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2533.7916946411133, |
| "epoch": 0.4034285714285714, |
| "grad_norm": 0.07949981093406677, |
| "kl": 4.393863491714001e-05, |
| "lambda_div_used": 0.6204324588179588, |
| "learning_rate": 3.195807108082429e-07, |
| "loss": 0.0456, |
| "reward": -0.12379613053053617, |
| "reward_after_mean": -0.12379613053053617, |
| "reward_after_std": 0.6454745382070541, |
| "reward_before_mean": 0.203694608528167, |
| "reward_before_std": 0.5591385969892144, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.327490733936429, |
| "reward_change_min": -0.5519858598709106, |
| "reward_change_std": 0.1984806014224887, |
| "reward_std": 0.6454745400696993, |
| "rewards/accuracy_reward": 0.2500000037252903, |
| "rewards/cosine_scaled_reward": -0.0463053984567523, |
| "step": 353 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1914.562515258789, |
| "epoch": 0.4045714285714286, |
| "grad_norm": 0.07853464037179947, |
| "kl": 2.3346394300460815e-05, |
| "lambda_div_used": 0.5937526449561119, |
| "learning_rate": 3.168878457820915e-07, |
| "loss": -0.0345, |
| "reward": 0.036306386813521385, |
| "reward_after_mean": 0.036306386813521385, |
| "reward_after_std": 0.5187021996825933, |
| "reward_before_mean": 0.4854283039458096, |
| "reward_before_std": 0.42800275422632694, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4491219110786915, |
| "reward_change_min": -0.637521255761385, |
| "reward_change_std": 0.252533134073019, |
| "reward_std": 0.5187022183090448, |
| "rewards/accuracy_reward": 0.33333334140479565, |
| "rewards/cosine_scaled_reward": 0.15209495089948177, |
| "step": 354 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2335.8958740234375, |
| "epoch": 0.4057142857142857, |
| "grad_norm": 0.09039249271154404, |
| "kl": 2.7535483241081238e-05, |
| "lambda_div_used": 0.6392721608281136, |
| "learning_rate": 3.142063423134644e-07, |
| "loss": 0.0158, |
| "reward": 0.017912205308675766, |
| "reward_after_mean": 0.017912205308675766, |
| "reward_after_std": 0.6705970745533705, |
| "reward_before_mean": 0.355070261284709, |
| "reward_before_std": 0.6396346259862185, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.33715808019042015, |
| "reward_change_min": -0.5581717267632484, |
| "reward_change_std": 0.21758130192756653, |
| "reward_std": 0.6705970987677574, |
| "rewards/accuracy_reward": 0.354166679084301, |
| "rewards/cosine_scaled_reward": 0.0009035973343998194, |
| "step": 355 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2585.4791870117188, |
| "epoch": 0.40685714285714286, |
| "grad_norm": 0.06332956254482269, |
| "kl": 1.6089528799057007e-05, |
| "lambda_div_used": 0.6472217217087746, |
| "learning_rate": 3.115363310950578e-07, |
| "loss": -0.0329, |
| "reward": -0.00402236171066761, |
| "reward_after_mean": -0.00402236171066761, |
| "reward_after_std": 0.6937260664999485, |
| "reward_before_mean": 0.2978215580806136, |
| "reward_before_std": 0.6849680617451668, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3018439058214426, |
| "reward_change_min": -0.4991762898862362, |
| "reward_change_std": 0.2024542335420847, |
| "reward_std": 0.6937260907143354, |
| "rewards/accuracy_reward": 0.2916666753590107, |
| "rewards/cosine_scaled_reward": 0.00615486316382885, |
| "step": 356 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3264.8125610351562, |
| "epoch": 0.408, |
| "grad_norm": 0.057182345539331436, |
| "kl": 2.91336327791214e-05, |
| "lambda_div_used": 0.5862653851509094, |
| "learning_rate": 3.0887794225945143e-07, |
| "loss": 0.0104, |
| "reward": -0.31888771802186966, |
| "reward_after_mean": -0.31888771802186966, |
| "reward_after_std": 0.43322005309164524, |
| "reward_before_mean": -0.049007685855031013, |
| "reward_before_std": 0.39840223640203476, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2698800265789032, |
| "reward_change_min": -0.48054200410842896, |
| "reward_change_std": 0.1758509548380971, |
| "reward_std": 0.43322005309164524, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.1531743509694934, |
| "step": 357 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3050.2500534057617, |
| "epoch": 0.40914285714285714, |
| "grad_norm": 0.05330098420381546, |
| "kl": 2.4370267055928707e-05, |
| "lambda_div_used": 0.6431510671973228, |
| "learning_rate": 3.062313053727671e-07, |
| "loss": 0.003, |
| "reward": -0.07201657444238663, |
| "reward_after_mean": -0.07201657444238663, |
| "reward_after_std": 0.688179362565279, |
| "reward_before_mean": 0.2027557131368667, |
| "reward_before_std": 0.6673171781003475, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27477228827774525, |
| "reward_change_min": -0.4944618083536625, |
| "reward_change_std": 0.18934499006718397, |
| "reward_std": 0.6881793700158596, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/cosine_scaled_reward": -0.026410957798361778, |
| "step": 358 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2671.937530517578, |
| "epoch": 0.4102857142857143, |
| "grad_norm": 0.07592163234949112, |
| "kl": 3.884732723236084e-05, |
| "lambda_div_used": 0.5795001536607742, |
| "learning_rate": 3.0359654942835247e-07, |
| "loss": 0.0464, |
| "reward": -0.2641681991517544, |
| "reward_after_mean": -0.2641681991517544, |
| "reward_after_std": 0.4112328961491585, |
| "reward_before_mean": 0.03885669261217117, |
| "reward_before_std": 0.36149344593286514, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.30302487686276436, |
| "reward_change_min": -0.4962599165737629, |
| "reward_change_std": 0.18252001702785492, |
| "reward_std": 0.4112329035997391, |
| "rewards/accuracy_reward": 0.1458333395421505, |
| "rewards/cosine_scaled_reward": -0.10697665251791477, |
| "step": 359 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2798.937530517578, |
| "epoch": 0.4114285714285714, |
| "grad_norm": 0.06665439158678055, |
| "kl": 3.1637027859687805e-05, |
| "lambda_div_used": 0.6062901616096497, |
| "learning_rate": 3.0097380284049523e-07, |
| "loss": -0.0029, |
| "reward": -0.0033570118248462677, |
| "reward_after_mean": -0.0033570118248462677, |
| "reward_after_std": 0.5593161657452583, |
| "reward_before_mean": 0.4031880460679531, |
| "reward_before_std": 0.4958424214273691, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.40654509887099266, |
| "reward_change_min": -0.6473257802426815, |
| "reward_change_std": 0.25588439870625734, |
| "reward_std": 0.5593161787837744, |
| "rewards/accuracy_reward": 0.3333333358168602, |
| "rewards/cosine_scaled_reward": 0.0698547288775444, |
| "step": 360 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3150.625030517578, |
| "epoch": 0.4125714285714286, |
| "grad_norm": 0.06971946358680725, |
| "kl": 2.9014074243605137e-05, |
| "lambda_div_used": 0.6219506114721298, |
| "learning_rate": 2.9836319343816397e-07, |
| "loss": 0.0048, |
| "reward": -0.11981392558664083, |
| "reward_after_mean": -0.11981392558664083, |
| "reward_after_std": 0.5868565142154694, |
| "reward_before_mean": 0.17182117886841297, |
| "reward_before_std": 0.564973471686244, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29163510724902153, |
| "reward_change_min": -0.5129627995193005, |
| "reward_change_std": 0.19524423126131296, |
| "reward_std": 0.5868565402925014, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": -0.057345494627952576, |
| "step": 361 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1471.916706085205, |
| "epoch": 0.4137142857142857, |
| "grad_norm": 0.12450961768627167, |
| "kl": 2.4568289518356323e-05, |
| "lambda_div_used": 0.571638435125351, |
| "learning_rate": 2.9576484845877793e-07, |
| "loss": -0.0303, |
| "reward": 0.013313630130141973, |
| "reward_after_mean": 0.013313630130141973, |
| "reward_after_std": 0.4897063076496124, |
| "reward_before_mean": 0.505017813295126, |
| "reward_before_std": 0.32288316125050187, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4917041715234518, |
| "reward_change_min": -0.6615025624632835, |
| "reward_change_std": 0.25815817900002, |
| "reward_std": 0.4897063188254833, |
| "rewards/accuracy_reward": 0.37500000558793545, |
| "rewards/cosine_scaled_reward": 0.1300177900120616, |
| "step": 362 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2174.270881652832, |
| "epoch": 0.41485714285714287, |
| "grad_norm": 0.08419760316610336, |
| "kl": 4.357472062110901e-05, |
| "lambda_div_used": 0.6037088930606842, |
| "learning_rate": 2.931788945420058e-07, |
| "loss": 0.0176, |
| "reward": -0.09517315030097961, |
| "reward_after_mean": -0.09517315030097961, |
| "reward_after_std": 0.57970448769629, |
| "reward_before_mean": 0.282845395617187, |
| "reward_before_std": 0.4830307289958, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.37801856361329556, |
| "reward_change_min": -0.5875305943191051, |
| "reward_change_std": 0.22446841653436422, |
| "reward_std": 0.57970448769629, |
| "rewards/accuracy_reward": 0.25000000186264515, |
| "rewards/cosine_scaled_reward": 0.03284538397565484, |
| "step": 363 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2614.9167098999023, |
| "epoch": 0.416, |
| "grad_norm": 0.07822614908218384, |
| "kl": 2.289540134370327e-05, |
| "lambda_div_used": 0.5602610111236572, |
| "learning_rate": 2.9060545772359305e-07, |
| "loss": 0.0079, |
| "reward": -0.4077296704053879, |
| "reward_after_mean": -0.4077296704053879, |
| "reward_after_std": 0.3345623780041933, |
| "reward_before_mean": -0.1330121699720621, |
| "reward_before_std": 0.27498441375792027, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2747174873948097, |
| "reward_change_min": -0.4260003827512264, |
| "reward_change_std": 0.16245513781905174, |
| "reward_std": 0.3345623817294836, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.195512181147933, |
| "step": 364 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3003.604202270508, |
| "epoch": 0.41714285714285715, |
| "grad_norm": 0.04828161001205444, |
| "kl": 6.87967985868454e-06, |
| "lambda_div_used": 0.6036554351449013, |
| "learning_rate": 2.8804466342921987e-07, |
| "loss": 0.0555, |
| "reward": -0.31679424038156867, |
| "reward_after_mean": -0.31679424038156867, |
| "reward_after_std": 0.5247047282755375, |
| "reward_before_mean": -0.07733823172748089, |
| "reward_before_std": 0.4768796032294631, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.23945600911974907, |
| "reward_change_min": -0.42952996119856834, |
| "reward_change_std": 0.15284609980881214, |
| "reward_std": 0.5247047450393438, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.18150490219704807, |
| "step": 365 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1893.00004196167, |
| "epoch": 0.41828571428571426, |
| "grad_norm": 0.08456390351057053, |
| "kl": 1.6473233699798584e-05, |
| "lambda_div_used": 0.6091032102704048, |
| "learning_rate": 2.854966364683872e-07, |
| "loss": -0.0195, |
| "reward": 0.005720527842640877, |
| "reward_after_mean": 0.005720527842640877, |
| "reward_after_std": 0.6514943428337574, |
| "reward_before_mean": 0.43899719044566154, |
| "reward_before_std": 0.49832448456436396, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4332766607403755, |
| "reward_change_min": -0.6058122254908085, |
| "reward_change_std": 0.23482245951890945, |
| "reward_std": 0.6514943763613701, |
| "rewards/accuracy_reward": 0.39583333767950535, |
| "rewards/cosine_scaled_reward": 0.043163834139704704, |
| "step": 366 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2780.937515258789, |
| "epoch": 0.41942857142857143, |
| "grad_norm": 0.06994574517011642, |
| "kl": 2.946704626083374e-05, |
| "lambda_div_used": 0.6079575195908546, |
| "learning_rate": 2.829615010283344e-07, |
| "loss": 0.0087, |
| "reward": -0.16347008850425482, |
| "reward_after_mean": -0.16347008850425482, |
| "reward_after_std": 0.5466162711381912, |
| "reward_before_mean": 0.1403359491378069, |
| "reward_before_std": 0.49341694079339504, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.30380604416131973, |
| "reward_change_min": -0.4883100688457489, |
| "reward_change_std": 0.18378268275409937, |
| "reward_std": 0.5466162823140621, |
| "rewards/accuracy_reward": 0.1875000074505806, |
| "rewards/cosine_scaled_reward": -0.0471640374744311, |
| "step": 367 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2967.041679382324, |
| "epoch": 0.4205714285714286, |
| "grad_norm": 0.07414139062166214, |
| "kl": 4.155188798904419e-05, |
| "lambda_div_used": 0.5803077146410942, |
| "learning_rate": 2.8043938066798645e-07, |
| "loss": 0.0504, |
| "reward": -0.27402403950691223, |
| "reward_after_mean": -0.27402403950691223, |
| "reward_after_std": 0.4059589561074972, |
| "reward_before_mean": 0.008191026747226715, |
| "reward_before_std": 0.371637674048543, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28221505135297775, |
| "reward_change_min": -0.44202784821391106, |
| "reward_change_std": 0.17603088915348053, |
| "reward_std": 0.40595896542072296, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.15847563743591309, |
| "step": 368 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2863.1667404174805, |
| "epoch": 0.4217142857142857, |
| "grad_norm": 0.07171155512332916, |
| "kl": 2.0891427993774414e-05, |
| "lambda_div_used": 0.6472674459218979, |
| "learning_rate": 2.7793039831193133e-07, |
| "loss": -0.0044, |
| "reward": -0.04304695222526789, |
| "reward_after_mean": -0.04304695222526789, |
| "reward_after_std": 0.687617726624012, |
| "reward_before_mean": 0.25146727729588747, |
| "reward_before_std": 0.6888436311855912, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29451421834528446, |
| "reward_change_min": -0.6169496960937977, |
| "reward_change_std": 0.22693553566932678, |
| "reward_std": 0.6876177359372377, |
| "rewards/accuracy_reward": 0.2500000037252903, |
| "rewards/cosine_scaled_reward": 0.0014672745019197464, |
| "step": 369 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3261.2291870117188, |
| "epoch": 0.4228571428571429, |
| "grad_norm": 0.058094967156648636, |
| "kl": 2.1554529666900635e-05, |
| "lambda_div_used": 0.5657742545008659, |
| "learning_rate": 2.7543467624442956e-07, |
| "loss": 0.0031, |
| "reward": -0.2483972143381834, |
| "reward_after_mean": -0.2483972143381834, |
| "reward_after_std": 0.4499538466334343, |
| "reward_before_mean": 0.13386818021535873, |
| "reward_before_std": 0.29791492875665426, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.38226539455354214, |
| "reward_change_min": -0.51585279032588, |
| "reward_change_std": 0.19617348536849022, |
| "reward_std": 0.4499538540840149, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.03279848676174879, |
| "step": 370 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1885.645866394043, |
| "epoch": 0.424, |
| "grad_norm": 0.12381042540073395, |
| "kl": 1.5240773791447282e-05, |
| "lambda_div_used": 0.5325883999466896, |
| "learning_rate": 2.729523361034538e-07, |
| "loss": -0.0304, |
| "reward": -0.14332129061222076, |
| "reward_after_mean": -0.14332129061222076, |
| "reward_after_std": 0.3713335506618023, |
| "reward_before_mean": 0.3775896169245243, |
| "reward_before_std": 0.14313361048698425, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5209108907729387, |
| "reward_change_min": -0.6799871250987053, |
| "reward_change_std": 0.26304140500724316, |
| "reward_std": 0.3713335543870926, |
| "rewards/accuracy_reward": 0.375, |
| "rewards/cosine_scaled_reward": 0.0025896020233631134, |
| "step": 371 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3061.6250228881836, |
| "epoch": 0.42514285714285716, |
| "grad_norm": 0.0565616711974144, |
| "kl": 1.3771001249551773e-05, |
| "lambda_div_used": 0.6153379678726196, |
| "learning_rate": 2.7048349887476037e-07, |
| "loss": 0.0019, |
| "reward": 0.08842738252133131, |
| "reward_after_mean": 0.08842738252133131, |
| "reward_after_std": 0.6899000462144613, |
| "reward_before_mean": 0.5513297475408763, |
| "reward_before_std": 0.5352799613028765, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.46290238574147224, |
| "reward_change_min": -0.6905807815492153, |
| "reward_change_std": 0.26435263454914093, |
| "reward_std": 0.6899000480771065, |
| "rewards/accuracy_reward": 0.3958333358168602, |
| "rewards/cosine_scaled_reward": 0.15549640776589513, |
| "step": 372 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1999.791690826416, |
| "epoch": 0.42628571428571427, |
| "grad_norm": 0.11289634555578232, |
| "kl": 2.6244670152664185e-05, |
| "lambda_div_used": 0.5901408270001411, |
| "learning_rate": 2.6802828488599294e-07, |
| "loss": -0.0028, |
| "reward": -0.25084885116666555, |
| "reward_after_mean": -0.25084885116666555, |
| "reward_after_std": 0.47954990342259407, |
| "reward_before_mean": 0.03629123326390982, |
| "reward_before_std": 0.41134614683687687, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28714008443057537, |
| "reward_change_min": -0.4200817756354809, |
| "reward_change_std": 0.1626526527106762, |
| "reward_std": 0.47954992204904556, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.08870877418667078, |
| "step": 373 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2700.791717529297, |
| "epoch": 0.42742857142857144, |
| "grad_norm": 0.0688440129160881, |
| "kl": 2.674013376235962e-05, |
| "lambda_div_used": 0.6036395579576492, |
| "learning_rate": 2.655868138008171e-07, |
| "loss": -0.0525, |
| "reward": -0.17511339485645294, |
| "reward_after_mean": -0.17511339485645294, |
| "reward_after_std": 0.5073369853198528, |
| "reward_before_mean": 0.13348117470741272, |
| "reward_before_std": 0.48259105905890465, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.30859458073973656, |
| "reward_change_min": -0.5309739634394646, |
| "reward_change_std": 0.20630291104316711, |
| "reward_std": 0.5073370076715946, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/cosine_scaled_reward": -0.07485215552151203, |
| "step": 374 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3033.791702270508, |
| "epoch": 0.42857142857142855, |
| "grad_norm": 0.05866050720214844, |
| "kl": 1.8252991139888763e-05, |
| "lambda_div_used": 0.6017400398850441, |
| "learning_rate": 2.631592046130896e-07, |
| "loss": 0.0477, |
| "reward": -0.18571221362799406, |
| "reward_after_mean": -0.18571221362799406, |
| "reward_after_std": 0.5023255608975887, |
| "reward_before_mean": 0.11062880232930183, |
| "reward_before_std": 0.46861019916832447, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2963410019874573, |
| "reward_change_min": -0.45132842287421227, |
| "reward_change_std": 0.1817708509042859, |
| "reward_std": 0.5023255832493305, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/cosine_scaled_reward": -0.097704553976655, |
| "step": 375 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2246.770851135254, |
| "epoch": 0.4297142857142857, |
| "grad_norm": 0.08337263762950897, |
| "kl": 8.605420589447021e-06, |
| "lambda_div_used": 0.5853807479143143, |
| "learning_rate": 2.6074557564105724e-07, |
| "loss": 0.0128, |
| "reward": -0.3334618601948023, |
| "reward_after_mean": -0.3334618601948023, |
| "reward_after_std": 0.4486316703259945, |
| "reward_before_mean": -0.05933803477091715, |
| "reward_before_std": 0.3892780668102205, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2741238009184599, |
| "reward_change_min": -0.46821001917123795, |
| "reward_change_std": 0.1663337228819728, |
| "reward_std": 0.44863167591392994, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.163504708558321, |
| "step": 376 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3488.3541870117188, |
| "epoch": 0.4308571428571429, |
| "grad_norm": 0.055966660380363464, |
| "kl": 2.5756657123565674e-05, |
| "lambda_div_used": 0.5742340236902237, |
| "learning_rate": 2.583460445215911e-07, |
| "loss": -0.0032, |
| "reward": -0.3738445993512869, |
| "reward_after_mean": -0.3738445993512869, |
| "reward_after_std": 0.39967909548431635, |
| "reward_before_mean": -0.10715527582215145, |
| "reward_before_std": 0.33878533728420734, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26668932288885117, |
| "reward_change_min": -0.4109174869954586, |
| "reward_change_std": 0.15686567965894938, |
| "reward_std": 0.39967911317944527, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.19048861227929592, |
| "step": 377 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2134.3333740234375, |
| "epoch": 0.432, |
| "grad_norm": 0.13181807100772858, |
| "kl": 2.8654932975769043e-05, |
| "lambda_div_used": 0.6004306748509407, |
| "learning_rate": 2.5596072820445254e-07, |
| "loss": -0.0331, |
| "reward": -0.06989812850952148, |
| "reward_after_mean": -0.06989812850952148, |
| "reward_after_std": 0.5459046512842178, |
| "reward_before_mean": 0.31881395215168595, |
| "reward_before_std": 0.46263560838997364, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3887120857834816, |
| "reward_change_min": -0.5917952992022038, |
| "reward_change_std": 0.23005317710340023, |
| "reward_std": 0.5459046605974436, |
| "rewards/accuracy_reward": 0.29166666977107525, |
| "rewards/cosine_scaled_reward": 0.027147281914949417, |
| "step": 378 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3121.0416870117188, |
| "epoch": 0.43314285714285716, |
| "grad_norm": 0.07427296042442322, |
| "kl": 1.5733763575553894e-05, |
| "lambda_div_used": 0.5833418369293213, |
| "learning_rate": 2.5358974294659373e-07, |
| "loss": -0.0048, |
| "reward": -0.29101302847266197, |
| "reward_after_mean": -0.29101302847266197, |
| "reward_after_std": 0.4290795102715492, |
| "reward_before_mean": -0.009382706135511398, |
| "reward_before_std": 0.38597164303064346, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2816303465515375, |
| "reward_change_min": -0.47839244455099106, |
| "reward_change_std": 0.17911072820425034, |
| "reward_std": 0.4290795363485813, |
| "rewards/accuracy_reward": 0.125, |
| "rewards/cosine_scaled_reward": -0.13438269309699535, |
| "step": 379 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2823.416702270508, |
| "epoch": 0.4342857142857143, |
| "grad_norm": 0.06611248105764389, |
| "kl": 2.2601336240768433e-05, |
| "lambda_div_used": 0.6066281795501709, |
| "learning_rate": 2.512332043064913e-07, |
| "loss": 0.0533, |
| "reward": -0.17613293696194887, |
| "reward_after_mean": -0.17613293696194887, |
| "reward_after_std": 0.495351817458868, |
| "reward_before_mean": 0.1346770692616701, |
| "reward_before_std": 0.48867712169885635, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.31081000342965126, |
| "reward_change_min": -0.5689719989895821, |
| "reward_change_std": 0.21053248085081577, |
| "reward_std": 0.4953518286347389, |
| "rewards/accuracy_reward": 0.1875000074505806, |
| "rewards/cosine_scaled_reward": -0.05282294252538122, |
| "step": 380 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2983.7916984558105, |
| "epoch": 0.43542857142857144, |
| "grad_norm": 0.08519254624843597, |
| "kl": 3.2504089176654816e-05, |
| "lambda_div_used": 0.6109358817338943, |
| "learning_rate": 2.488912271385139e-07, |
| "loss": 0.0165, |
| "reward": -0.2218069350346923, |
| "reward_after_mean": -0.2218069350346923, |
| "reward_after_std": 0.5642109382897615, |
| "reward_before_mean": 0.040106164291501045, |
| "reward_before_std": 0.5099438140168786, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.261913089081645, |
| "reward_change_min": -0.45296039059758186, |
| "reward_change_std": 0.1642473293468356, |
| "reward_std": 0.5642109606415033, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.10572716826573014, |
| "step": 381 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2500.437545776367, |
| "epoch": 0.43657142857142855, |
| "grad_norm": 0.07325682789087296, |
| "kl": 3.65767627954483e-05, |
| "lambda_div_used": 0.5600408837199211, |
| "learning_rate": 2.465639255873246e-07, |
| "loss": -0.0096, |
| "reward": -0.4078812226653099, |
| "reward_after_mean": -0.4078812226653099, |
| "reward_after_std": 0.3340430334210396, |
| "reward_before_mean": -0.13890772312879562, |
| "reward_before_std": 0.27443209011107683, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26897350139915943, |
| "reward_change_min": -0.42554087564349174, |
| "reward_change_std": 0.15754834469407797, |
| "reward_std": 0.3340430427342653, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.20140772499144077, |
| "step": 382 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2824.291717529297, |
| "epoch": 0.4377142857142857, |
| "grad_norm": 0.08796168118715286, |
| "kl": 4.8138201236724854e-05, |
| "lambda_div_used": 0.5781937688589096, |
| "learning_rate": 2.4425141308231765e-07, |
| "loss": -0.0086, |
| "reward": -0.16291768848896027, |
| "reward_after_mean": -0.16291768848896027, |
| "reward_after_std": 0.387674568220973, |
| "reward_before_mean": 0.1924248207360506, |
| "reward_before_std": 0.35597400926053524, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3553424943238497, |
| "reward_change_min": -0.5255323797464371, |
| "reward_change_std": 0.21135967783629894, |
| "reward_std": 0.3876745719462633, |
| "rewards/accuracy_reward": 0.2083333432674408, |
| "rewards/cosine_scaled_reward": -0.01590852066874504, |
| "step": 383 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2347.6042404174805, |
| "epoch": 0.43885714285714283, |
| "grad_norm": 0.09212023764848709, |
| "kl": 2.508983016014099e-05, |
| "lambda_div_used": 0.6615950018167496, |
| "learning_rate": 2.4195380233209006e-07, |
| "loss": 0.0408, |
| "reward": 0.27819300815463066, |
| "reward_after_mean": 0.27819300815463066, |
| "reward_after_std": 0.7268820777535439, |
| "reward_before_mean": 0.6785084716975689, |
| "reward_before_std": 0.7540020374581218, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4003154691308737, |
| "reward_change_min": -0.6902611702680588, |
| "reward_change_std": 0.283959056250751, |
| "reward_std": 0.7268821019679308, |
| "rewards/accuracy_reward": 0.4583333432674408, |
| "rewards/cosine_scaled_reward": 0.22017512656748295, |
| "step": 384 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2711.958396911621, |
| "epoch": 0.44, |
| "grad_norm": 0.07038652151823044, |
| "kl": 1.942366361618042e-05, |
| "lambda_div_used": 0.6165113672614098, |
| "learning_rate": 2.3967120531894857e-07, |
| "loss": 0.012, |
| "reward": -0.25626325886696577, |
| "reward_after_mean": -0.25626325886696577, |
| "reward_after_std": 0.5892766248434782, |
| "reward_before_mean": -0.0074533987790346146, |
| "reward_before_std": 0.5360177559778094, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24880987033247948, |
| "reward_change_min": -0.44854912906885147, |
| "reward_change_std": 0.1595793990418315, |
| "reward_std": 0.5892766322940588, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.13245339877903461, |
| "step": 385 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2889.1042404174805, |
| "epoch": 0.44114285714285717, |
| "grad_norm": 0.060970280319452286, |
| "kl": 1.8231570720672607e-05, |
| "lambda_div_used": 0.5940364003181458, |
| "learning_rate": 2.374037332934512e-07, |
| "loss": -0.0287, |
| "reward": -0.24576714914292097, |
| "reward_after_mean": -0.24576714914292097, |
| "reward_after_std": 0.4883785657584667, |
| "reward_before_mean": 0.04634229093790054, |
| "reward_before_std": 0.4279829766601324, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29210945032536983, |
| "reward_change_min": -0.4432514049112797, |
| "reward_change_std": 0.17009661067277193, |
| "reward_std": 0.48837856762111187, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.07865770626813173, |
| "step": 386 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2997.3125, |
| "epoch": 0.4422857142857143, |
| "grad_norm": 0.07781542837619781, |
| "kl": 1.3821758329868317e-05, |
| "lambda_div_used": 0.5551474094390869, |
| "learning_rate": 2.3515149676898552e-07, |
| "loss": -0.0529, |
| "reward": -0.38486793637275696, |
| "reward_after_mean": -0.38486793637275696, |
| "reward_after_std": 0.31045267172157764, |
| "reward_before_mean": -0.10442159324884415, |
| "reward_before_std": 0.24840335873886943, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28044634498655796, |
| "reward_change_min": -0.4241391494870186, |
| "reward_change_std": 0.15805031638592482, |
| "reward_std": 0.31045267917215824, |
| "rewards/accuracy_reward": 0.1041666716337204, |
| "rewards/cosine_scaled_reward": -0.20858826488256454, |
| "step": 387 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2572.2708587646484, |
| "epoch": 0.44342857142857145, |
| "grad_norm": 0.06563723832368851, |
| "kl": 2.9848888516426086e-06, |
| "lambda_div_used": 0.5865650475025177, |
| "learning_rate": 2.3291460551638237e-07, |
| "loss": 0.0407, |
| "reward": -0.15434425324201584, |
| "reward_after_mean": -0.15434425324201584, |
| "reward_after_std": 0.47601727209985256, |
| "reward_before_mean": 0.20021704956889153, |
| "reward_before_std": 0.39916726760566235, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.354561323300004, |
| "reward_change_min": -0.5467559210956097, |
| "reward_change_std": 0.21171134896576405, |
| "reward_std": 0.47601727209985256, |
| "rewards/accuracy_reward": 0.2708333358168602, |
| "rewards/cosine_scaled_reward": -0.07061627879738808, |
| "step": 388 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2650.937545776367, |
| "epoch": 0.44457142857142856, |
| "grad_norm": 0.07840488851070404, |
| "kl": 1.73286534845829e-05, |
| "lambda_div_used": 0.6045578718185425, |
| "learning_rate": 2.306931685585657e-07, |
| "loss": -0.0394, |
| "reward": -0.26122746989130974, |
| "reward_after_mean": -0.26122746989130974, |
| "reward_after_std": 0.5210577324032784, |
| "reward_before_mean": 0.002998221665620804, |
| "reward_before_std": 0.486898148432374, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26422569528222084, |
| "reward_change_min": -0.4951511509716511, |
| "reward_change_std": 0.18348107766360044, |
| "reward_std": 0.5210577566176653, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.1428351059439592, |
| "step": 389 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3007.666679382324, |
| "epoch": 0.44571428571428573, |
| "grad_norm": 0.07837636768817902, |
| "kl": 1.6082078218460083e-05, |
| "lambda_div_used": 0.61600511521101, |
| "learning_rate": 2.2848729416523859e-07, |
| "loss": 0.0713, |
| "reward": -0.18647840432822704, |
| "reward_after_mean": -0.18647840432822704, |
| "reward_after_std": 0.5758322961628437, |
| "reward_before_mean": 0.08954079262912273, |
| "reward_before_std": 0.5347468825057149, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27601918019354343, |
| "reward_change_min": -0.47189946845173836, |
| "reward_change_std": 0.17452176753431559, |
| "reward_std": 0.5758323054760695, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.07712590089067817, |
| "step": 390 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2716.708351135254, |
| "epoch": 0.44685714285714284, |
| "grad_norm": 0.09803462773561478, |
| "kl": 2.5559216737747192e-05, |
| "lambda_div_used": 0.6362191960215569, |
| "learning_rate": 2.2629708984760706e-07, |
| "loss": 0.0166, |
| "reward": -0.013867436617147177, |
| "reward_after_mean": -0.013867436617147177, |
| "reward_after_std": 0.6719868443906307, |
| "reward_before_mean": 0.30256712157279253, |
| "reward_before_std": 0.6324740117415786, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3164345696568489, |
| "reward_change_min": -0.5261501334607601, |
| "reward_change_std": 0.20623697619885206, |
| "reward_std": 0.6719868592917919, |
| "rewards/accuracy_reward": 0.2708333432674408, |
| "rewards/cosine_scaled_reward": 0.03173377411440015, |
| "step": 391 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2130.9791984558105, |
| "epoch": 0.448, |
| "grad_norm": 0.11540649086236954, |
| "kl": 1.8610619008541107e-05, |
| "lambda_div_used": 0.6027960926294327, |
| "learning_rate": 2.2412266235313973e-07, |
| "loss": -0.0919, |
| "reward": -0.16835473664104939, |
| "reward_after_mean": -0.16835473664104939, |
| "reward_after_std": 0.5122340489178896, |
| "reward_before_mean": 0.1393747702240944, |
| "reward_before_std": 0.4794600326567888, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3077295087277889, |
| "reward_change_min": -0.4898943528532982, |
| "reward_change_std": 0.1954572731629014, |
| "reward_std": 0.5122340768575668, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/cosine_scaled_reward": -0.027291906299069524, |
| "step": 392 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2649.562530517578, |
| "epoch": 0.4491428571428571, |
| "grad_norm": 0.07226411253213882, |
| "kl": 2.648681402206421e-05, |
| "lambda_div_used": 0.5953627824783325, |
| "learning_rate": 2.2196411766036487e-07, |
| "loss": -0.067, |
| "reward": -0.20140517689287663, |
| "reward_after_mean": -0.20140517689287663, |
| "reward_after_std": 0.4766168761998415, |
| "reward_before_mean": 0.09705937840044498, |
| "reward_before_std": 0.4375506564974785, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2984645199030638, |
| "reward_change_min": -0.47758448868989944, |
| "reward_change_std": 0.18242334388196468, |
| "reward_std": 0.47661688551306725, |
| "rewards/accuracy_reward": 0.18750000558793545, |
| "rewards/cosine_scaled_reward": -0.09044064581394196, |
| "step": 393 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3104.1458740234375, |
| "epoch": 0.4502857142857143, |
| "grad_norm": 0.07446404546499252, |
| "kl": 2.555176615715027e-05, |
| "lambda_div_used": 0.5502220839262009, |
| "learning_rate": 2.1982156097370557e-07, |
| "loss": -0.1089, |
| "reward": -0.46874738670885563, |
| "reward_after_mean": -0.46874738670885563, |
| "reward_after_std": 0.30342659167945385, |
| "reward_before_mean": -0.20392105542123318, |
| "reward_before_std": 0.22309745661914349, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2648263294249773, |
| "reward_change_min": -0.3818593733012676, |
| "reward_change_std": 0.14172559697180986, |
| "reward_std": 0.30342659167945385, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.22475438751280308, |
| "step": 394 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2284.6458435058594, |
| "epoch": 0.4514285714285714, |
| "grad_norm": 0.12902304530143738, |
| "kl": 4.9501657485961914e-05, |
| "lambda_div_used": 0.5602920204401016, |
| "learning_rate": 2.1769509671835223e-07, |
| "loss": -0.1675, |
| "reward": -0.3165151756256819, |
| "reward_after_mean": -0.3165151756256819, |
| "reward_after_std": 0.3793674483895302, |
| "reward_before_mean": 0.0216768067330122, |
| "reward_before_std": 0.27139727398753166, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3381919823586941, |
| "reward_change_min": -0.48955149203538895, |
| "reward_change_std": 0.17990652937442064, |
| "reward_std": 0.3793674521148205, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.12415653187781572, |
| "step": 395 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3048.1458587646484, |
| "epoch": 0.45257142857142857, |
| "grad_norm": 0.054839495569467545, |
| "kl": 7.04331323504448e-05, |
| "lambda_div_used": 0.6417915895581245, |
| "learning_rate": 2.1558482853517253e-07, |
| "loss": 0.0192, |
| "reward": -0.09411877207458019, |
| "reward_after_mean": -0.09411877207458019, |
| "reward_after_std": 0.6742421016097069, |
| "reward_before_mean": 0.1871709941624431, |
| "reward_before_std": 0.659186695702374, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2812897562980652, |
| "reward_change_min": -0.5664796940982342, |
| "reward_change_std": 0.20405743923038244, |
| "reward_std": 0.6742421071976423, |
| "rewards/accuracy_reward": 0.20833333767950535, |
| "rewards/cosine_scaled_reward": -0.021162351593375206, |
| "step": 396 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2982.750030517578, |
| "epoch": 0.45371428571428574, |
| "grad_norm": 0.05798732861876488, |
| "kl": 2.0850449800491333e-05, |
| "lambda_div_used": 0.5765915215015411, |
| "learning_rate": 2.134908592756607e-07, |
| "loss": 0.0391, |
| "reward": -0.36006474308669567, |
| "reward_after_mean": -0.36006474308669567, |
| "reward_after_std": 0.40458301082253456, |
| "reward_before_mean": -0.08499279711395502, |
| "reward_before_std": 0.3492864612489939, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2750719413161278, |
| "reward_change_min": -0.4323217496275902, |
| "reward_change_std": 0.16262990981340408, |
| "reward_std": 0.40458302199840546, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.16832613572478294, |
| "step": 397 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2822.5209045410156, |
| "epoch": 0.45485714285714285, |
| "grad_norm": 0.06360434740781784, |
| "kl": 2.2810418158769608e-05, |
| "lambda_div_used": 0.6480221897363663, |
| "learning_rate": 2.1141329099692406e-07, |
| "loss": 0.0465, |
| "reward": -0.09229415841400623, |
| "reward_after_mean": -0.09229415841400623, |
| "reward_after_std": 0.7079674638807774, |
| "reward_before_mean": 0.1752314588520676, |
| "reward_before_std": 0.6917341919615865, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26752561889588833, |
| "reward_change_min": -0.5181693434715271, |
| "reward_change_std": 0.19102454278618097, |
| "reward_std": 0.7079674787819386, |
| "rewards/accuracy_reward": 0.2083333358168602, |
| "rewards/cosine_scaled_reward": -0.03310186788439751, |
| "step": 398 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2191.583351135254, |
| "epoch": 0.456, |
| "grad_norm": 0.07585066556930542, |
| "kl": 3.449246287345886e-05, |
| "lambda_div_used": 0.6071069464087486, |
| "learning_rate": 2.0935222495670968e-07, |
| "loss": -0.0428, |
| "reward": -0.06968086212873459, |
| "reward_after_mean": -0.06968086212873459, |
| "reward_after_std": 0.5623702071607113, |
| "reward_before_mean": 0.29000685061328113, |
| "reward_before_std": 0.4939764440059662, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.35968772508203983, |
| "reward_change_min": -0.5781176537275314, |
| "reward_change_std": 0.2176226656883955, |
| "reward_std": 0.5623702295124531, |
| "rewards/accuracy_reward": 0.27083333767950535, |
| "rewards/cosine_scaled_reward": 0.01917351223528385, |
| "step": 399 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1814.4375305175781, |
| "epoch": 0.45714285714285713, |
| "grad_norm": 0.09906096011400223, |
| "kl": 3.718771040439606e-05, |
| "lambda_div_used": 0.615766242146492, |
| "learning_rate": 2.0730776160846853e-07, |
| "loss": -0.0203, |
| "reward": 0.023047026246786118, |
| "reward_after_mean": 0.023047026246786118, |
| "reward_after_std": 0.6236102003604174, |
| "reward_before_mean": 0.43256790889427066, |
| "reward_before_std": 0.5322064086794853, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.40952087566256523, |
| "reward_change_min": -0.6220400035381317, |
| "reward_change_std": 0.2354581467807293, |
| "reward_std": 0.62361023388803, |
| "rewards/accuracy_reward": 0.3125000074505806, |
| "rewards/cosine_scaled_reward": 0.12006791215389967, |
| "step": 400 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3091.7916870117188, |
| "epoch": 0.4582857142857143, |
| "grad_norm": 0.05475914105772972, |
| "kl": 3.5446137189865112e-06, |
| "lambda_div_used": 0.562153548002243, |
| "learning_rate": 2.0528000059645995e-07, |
| "loss": 0.0508, |
| "reward": -0.29909564182162285, |
| "reward_after_mean": -0.29909564182162285, |
| "reward_after_std": 0.3837758805602789, |
| "reward_before_mean": 0.04575100168585777, |
| "reward_before_std": 0.28164876997470856, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3448466807603836, |
| "reward_change_min": -0.5300594680011272, |
| "reward_change_std": 0.19381076097488403, |
| "reward_std": 0.38377588987350464, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.12091565364971757, |
| "step": 401 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2440.1458740234375, |
| "epoch": 0.4594285714285714, |
| "grad_norm": 0.11770489811897278, |
| "kl": 1.8846243619918823e-05, |
| "lambda_div_used": 0.5605455562472343, |
| "learning_rate": 2.032690407508949e-07, |
| "loss": -0.0261, |
| "reward": -0.2383374497294426, |
| "reward_after_mean": -0.2383374497294426, |
| "reward_after_std": 0.3794025480747223, |
| "reward_before_mean": 0.14294641837477684, |
| "reward_before_std": 0.27562023140490055, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.38128384202718735, |
| "reward_change_min": -0.5627436973154545, |
| "reward_change_std": 0.2136234436184168, |
| "reward_std": 0.3794025518000126, |
| "rewards/accuracy_reward": 0.1875, |
| "rewards/cosine_scaled_reward": -0.044553620740771294, |
| "step": 402 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1985.6875343322754, |
| "epoch": 0.4605714285714286, |
| "grad_norm": 0.10363903641700745, |
| "kl": 2.3838132619857788e-05, |
| "lambda_div_used": 0.5793934315443039, |
| "learning_rate": 2.0127498008311922e-07, |
| "loss": 0.003, |
| "reward": -0.17537187691777945, |
| "reward_after_mean": -0.17537187691777945, |
| "reward_after_std": 0.46652381494641304, |
| "reward_before_mean": 0.20323466695845127, |
| "reward_before_std": 0.3647587588056922, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.37860656157135963, |
| "reward_change_min": -0.5506669841706753, |
| "reward_change_std": 0.21334033645689487, |
| "reward_std": 0.4665238317102194, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": -0.025931986048817635, |
| "step": 403 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2137.375, |
| "epoch": 0.4617142857142857, |
| "grad_norm": 0.10717064887285233, |
| "kl": 3.7202611565589905e-05, |
| "lambda_div_used": 0.5993036478757858, |
| "learning_rate": 1.9929791578083655e-07, |
| "loss": -0.0043, |
| "reward": -0.053446926176548004, |
| "reward_after_mean": -0.053446926176548004, |
| "reward_after_std": 0.4612566214054823, |
| "reward_before_mean": 0.3198554217815399, |
| "reward_before_std": 0.4542539082467556, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3733023554086685, |
| "reward_change_min": -0.5774630047380924, |
| "reward_change_std": 0.2339334823191166, |
| "reward_std": 0.4612566400319338, |
| "rewards/accuracy_reward": 0.2708333432674408, |
| "rewards/cosine_scaled_reward": 0.04902207478880882, |
| "step": 404 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2319.5000228881836, |
| "epoch": 0.46285714285714286, |
| "grad_norm": 0.13891565799713135, |
| "kl": 2.9403716325759888e-05, |
| "lambda_div_used": 0.6237343773245811, |
| "learning_rate": 1.9733794420337213e-07, |
| "loss": -0.0602, |
| "reward": 0.1285000964999199, |
| "reward_after_mean": 0.1285000964999199, |
| "reward_after_std": 0.7067493386566639, |
| "reward_before_mean": 0.5776062086224556, |
| "reward_before_std": 0.5746421907097101, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.449106115847826, |
| "reward_change_min": -0.6593505516648293, |
| "reward_change_std": 0.263070298358798, |
| "reward_std": 0.7067493461072445, |
| "rewards/accuracy_reward": 0.4166666679084301, |
| "rewards/cosine_scaled_reward": 0.1609395444393158, |
| "step": 405 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2442.1458892822266, |
| "epoch": 0.464, |
| "grad_norm": 0.07689624279737473, |
| "kl": 8.527189493179321e-06, |
| "lambda_div_used": 0.6336105018854141, |
| "learning_rate": 1.9539516087697517e-07, |
| "loss": -0.0, |
| "reward": -0.11170937749557197, |
| "reward_after_mean": -0.11170937749557197, |
| "reward_after_std": 0.6453428398817778, |
| "reward_before_mean": 0.17541324836201966, |
| "reward_before_std": 0.6142212487757206, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2871226370334625, |
| "reward_change_min": -0.5107162520289421, |
| "reward_change_std": 0.19029081240296364, |
| "reward_std": 0.645342854782939, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/cosine_scaled_reward": -0.032920089550316334, |
| "step": 406 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2516.6041946411133, |
| "epoch": 0.46514285714285714, |
| "grad_norm": 0.09361904859542847, |
| "kl": 2.995133399963379e-05, |
| "lambda_div_used": 0.5926980003714561, |
| "learning_rate": 1.934696604901642e-07, |
| "loss": 0.011, |
| "reward": -0.16482560662552714, |
| "reward_after_mean": -0.16482560662552714, |
| "reward_after_std": 0.5236028637737036, |
| "reward_before_mean": 0.18611273169517517, |
| "reward_before_std": 0.4258856289088726, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.35093834809958935, |
| "reward_change_min": -0.5517234578728676, |
| "reward_change_std": 0.20658957865089178, |
| "reward_std": 0.5236028656363487, |
| "rewards/accuracy_reward": 0.25000000186264515, |
| "rewards/cosine_scaled_reward": -0.06388726737350225, |
| "step": 407 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2709.4791984558105, |
| "epoch": 0.4662857142857143, |
| "grad_norm": 0.09306048601865768, |
| "kl": 3.203703090548515e-05, |
| "lambda_div_used": 0.5583930537104607, |
| "learning_rate": 1.915615368891117e-07, |
| "loss": -0.111, |
| "reward": -0.17684245621785522, |
| "reward_after_mean": -0.17684245621785522, |
| "reward_after_std": 0.42921141162514687, |
| "reward_before_mean": 0.2615569829940796, |
| "reward_before_std": 0.26226984336972237, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.43839943781495094, |
| "reward_change_min": -0.6072936318814754, |
| "reward_change_std": 0.22694199439138174, |
| "reward_std": 0.42921141907572746, |
| "rewards/accuracy_reward": 0.27083333395421505, |
| "rewards/cosine_scaled_reward": -0.009276359807699919, |
| "step": 408 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3529.3541870117188, |
| "epoch": 0.4674285714285714, |
| "grad_norm": 0.048929836601018906, |
| "kl": 7.6089054346084595e-06, |
| "lambda_div_used": 0.6116252392530441, |
| "learning_rate": 1.8967088307307e-07, |
| "loss": 0.001, |
| "reward": -0.17062923312187195, |
| "reward_after_mean": -0.17062923312187195, |
| "reward_after_std": 0.5682655684649944, |
| "reward_before_mean": 0.12847105879336596, |
| "reward_before_std": 0.5097848381847143, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2991002984344959, |
| "reward_change_min": -0.43997345492243767, |
| "reward_change_std": 0.17078326642513275, |
| "reward_std": 0.5682655889540911, |
| "rewards/accuracy_reward": 0.16666667349636555, |
| "rewards/cosine_scaled_reward": -0.038195611676201224, |
| "step": 409 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2435.625045776367, |
| "epoch": 0.4685714285714286, |
| "grad_norm": 0.08304905891418457, |
| "kl": 1.5234574675559998e-05, |
| "lambda_div_used": 0.6268336623907089, |
| "learning_rate": 1.8779779118983867e-07, |
| "loss": 0.0499, |
| "reward": -0.03433734131976962, |
| "reward_after_mean": -0.03433734131976962, |
| "reward_after_std": 0.6416803412139416, |
| "reward_before_mean": 0.2960522407665849, |
| "reward_before_std": 0.5826444877311587, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3303895927965641, |
| "reward_change_min": -0.4872638136148453, |
| "reward_change_std": 0.19508948456496, |
| "reward_std": 0.6416803747415543, |
| "rewards/accuracy_reward": 0.25000000931322575, |
| "rewards/cosine_scaled_reward": 0.04605222793179564, |
| "step": 410 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3163.1041717529297, |
| "epoch": 0.4697142857142857, |
| "grad_norm": 0.06846357136964798, |
| "kl": 1.529604196548462e-05, |
| "lambda_div_used": 0.5874563306570053, |
| "learning_rate": 1.8594235253127372e-07, |
| "loss": 0.0214, |
| "reward": -0.22584301605820656, |
| "reward_after_mean": -0.22584301605820656, |
| "reward_after_std": 0.42311797849833965, |
| "reward_before_mean": 0.0774743240326643, |
| "reward_before_std": 0.4020446836948395, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3033173345029354, |
| "reward_change_min": -0.48705917969346046, |
| "reward_change_std": 0.1920422399416566, |
| "reward_std": 0.42311798594892025, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/cosine_scaled_reward": -0.08919236063957214, |
| "step": 411 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2888.125015258789, |
| "epoch": 0.47085714285714286, |
| "grad_norm": 0.06694348156452179, |
| "kl": 2.148747444152832e-05, |
| "lambda_div_used": 0.6005539819598198, |
| "learning_rate": 1.8410465752883758e-07, |
| "loss": 0.049, |
| "reward": -0.21726901549845934, |
| "reward_after_mean": -0.21726901549845934, |
| "reward_after_std": 0.48868509009480476, |
| "reward_before_mean": 0.08159955404698849, |
| "reward_before_std": 0.463420107960701, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2988685816526413, |
| "reward_change_min": -0.554147582501173, |
| "reward_change_std": 0.19932363275438547, |
| "reward_std": 0.48868510872125626, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.06423377990722656, |
| "step": 412 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2530.2708702087402, |
| "epoch": 0.472, |
| "grad_norm": 0.08418448269367218, |
| "kl": 2.208724617958069e-05, |
| "lambda_div_used": 0.6184473037719727, |
| "learning_rate": 1.822847957491922e-07, |
| "loss": 0.0462, |
| "reward": 0.05549921467900276, |
| "reward_after_mean": 0.05549921467900276, |
| "reward_after_std": 0.6010984163731337, |
| "reward_before_mean": 0.4671566132456064, |
| "reward_before_std": 0.546179112046957, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4116574004292488, |
| "reward_change_min": -0.6464647352695465, |
| "reward_change_std": 0.253952544182539, |
| "reward_std": 0.6010984499007463, |
| "rewards/accuracy_reward": 0.37500000931322575, |
| "rewards/cosine_scaled_reward": 0.09215660532936454, |
| "step": 413 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2948.562530517578, |
| "epoch": 0.47314285714285714, |
| "grad_norm": 0.06618716567754745, |
| "kl": 1.0458752512931824e-05, |
| "lambda_div_used": 0.6192215830087662, |
| "learning_rate": 1.804828558898332e-07, |
| "loss": -0.0483, |
| "reward": -0.21263186633586884, |
| "reward_after_mean": -0.21263186633586884, |
| "reward_after_std": 0.5998369809240103, |
| "reward_before_mean": 0.04798411298543215, |
| "reward_before_std": 0.555210480466485, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2606159746646881, |
| "reward_change_min": -0.4564817361533642, |
| "reward_change_std": 0.16899613942950964, |
| "reward_std": 0.5998369976878166, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.0978492172434926, |
| "step": 414 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3234.8125, |
| "epoch": 0.4742857142857143, |
| "grad_norm": 0.06085793301463127, |
| "kl": 2.8954818844795227e-05, |
| "lambda_div_used": 0.5570018589496613, |
| "learning_rate": 1.7869892577476722e-07, |
| "loss": 0.0073, |
| "reward": -0.46066740388050675, |
| "reward_after_mean": -0.46066740388050675, |
| "reward_after_std": 0.3419057931751013, |
| "reward_before_mean": -0.2106443401426077, |
| "reward_before_std": 0.2546057654544711, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2500230632722378, |
| "reward_change_min": -0.36453111097216606, |
| "reward_change_std": 0.1324605904519558, |
| "reward_std": 0.3419058118015528, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.23147768154740334, |
| "step": 415 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1748.9583892822266, |
| "epoch": 0.4754285714285714, |
| "grad_norm": 0.09120064228773117, |
| "kl": 8.981674909591675e-06, |
| "lambda_div_used": 0.6067373231053352, |
| "learning_rate": 1.7693309235023127e-07, |
| "loss": -0.0771, |
| "reward": -0.13937147613614798, |
| "reward_after_mean": -0.13937147613614798, |
| "reward_after_std": 0.6240962240844965, |
| "reward_before_mean": 0.20647221896797419, |
| "reward_before_std": 0.48884215706493706, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34584368020296097, |
| "reward_change_min": -0.5141387544572353, |
| "reward_change_std": 0.18731264490634203, |
| "reward_std": 0.6240962333977222, |
| "rewards/accuracy_reward": 0.2500000037252903, |
| "rewards/cosine_scaled_reward": -0.04352780181216076, |
| "step": 416 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3307.250015258789, |
| "epoch": 0.4765714285714286, |
| "grad_norm": 0.05766940861940384, |
| "kl": 3.1495699658989906e-05, |
| "lambda_div_used": 0.5544050261378288, |
| "learning_rate": 1.7518544168045524e-07, |
| "loss": -0.0019, |
| "reward": -0.4980150870978832, |
| "reward_after_mean": -0.4980150870978832, |
| "reward_after_std": 0.3123048096895218, |
| "reward_before_mean": -0.2511206082999706, |
| "reward_before_std": 0.24326619878411293, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24689447320997715, |
| "reward_change_min": -0.3710709176957607, |
| "reward_change_std": 0.13346257898956537, |
| "reward_std": 0.3123048171401024, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2719539441168308, |
| "step": 417 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2171.625015258789, |
| "epoch": 0.4777142857142857, |
| "grad_norm": 0.0854194164276123, |
| "kl": 2.8558075428009033e-05, |
| "lambda_div_used": 0.6033797562122345, |
| "learning_rate": 1.7345605894346726e-07, |
| "loss": 0.0073, |
| "reward": -0.0949536501429975, |
| "reward_after_mean": -0.0949536501429975, |
| "reward_after_std": 0.540143633261323, |
| "reward_before_mean": 0.25413690507411957, |
| "reward_before_std": 0.4725718079134822, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34909053333103657, |
| "reward_change_min": -0.5411153584718704, |
| "reward_change_std": 0.2063078135251999, |
| "reward_std": 0.5401436407119036, |
| "rewards/accuracy_reward": 0.29166667349636555, |
| "rewards/cosine_scaled_reward": -0.03752976981922984, |
| "step": 418 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2464.3750228881836, |
| "epoch": 0.47885714285714287, |
| "grad_norm": 0.08315848559141159, |
| "kl": 3.0182301998138428e-05, |
| "lambda_div_used": 0.5937136337161064, |
| "learning_rate": 1.7174502842694212e-07, |
| "loss": 0.03, |
| "reward": -0.27206650376319885, |
| "reward_after_mean": -0.27206650376319885, |
| "reward_after_std": 0.49108788557350636, |
| "reward_before_mean": 0.0038498505018651485, |
| "reward_before_std": 0.4298001816496253, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27591635659337044, |
| "reward_change_min": -0.4215012192726135, |
| "reward_change_std": 0.16396540496498346, |
| "reward_std": 0.49108790047466755, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.12115013878792524, |
| "step": 419 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1972.7083625793457, |
| "epoch": 0.48, |
| "grad_norm": 0.11145550012588501, |
| "kl": 3.493949770927429e-05, |
| "lambda_div_used": 0.5797194987535477, |
| "learning_rate": 1.7005243352409333e-07, |
| "loss": -0.0607, |
| "reward": -0.1652970388531685, |
| "reward_after_mean": -0.1652970388531685, |
| "reward_after_std": 0.4477591011673212, |
| "reward_before_mean": 0.18980162939988077, |
| "reward_before_std": 0.36517443507909775, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.35509867407381535, |
| "reward_change_min": -0.5067496970295906, |
| "reward_change_std": 0.19984195847064257, |
| "reward_std": 0.4477591197937727, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": -0.03936504106968641, |
| "step": 420 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3191.937515258789, |
| "epoch": 0.48114285714285715, |
| "grad_norm": 0.07170204073190689, |
| "kl": 3.11434268951416e-05, |
| "lambda_div_used": 0.5753046199679375, |
| "learning_rate": 1.6837835672960831e-07, |
| "loss": 0.0088, |
| "reward": -0.38796948781237006, |
| "reward_after_mean": -0.38796948781237006, |
| "reward_after_std": 0.4225973077118397, |
| "reward_before_mean": -0.12759371474385262, |
| "reward_before_std": 0.3405588921159506, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2603757604956627, |
| "reward_change_min": -0.37436668202281, |
| "reward_change_std": 0.13996374886482954, |
| "reward_std": 0.4225973132997751, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.16926039289683104, |
| "step": 421 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2776.437545776367, |
| "epoch": 0.48228571428571426, |
| "grad_norm": 0.06681375205516815, |
| "kl": 1.9788742065429688e-05, |
| "lambda_div_used": 0.5904401987791061, |
| "learning_rate": 1.6672287963562852e-07, |
| "loss": 0.0293, |
| "reward": -0.25396816432476044, |
| "reward_after_mean": -0.25396816432476044, |
| "reward_after_std": 0.4787884410470724, |
| "reward_before_mean": 0.03486193250864744, |
| "reward_before_std": 0.4093271289020777, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2888300847262144, |
| "reward_change_min": -0.43693453073501587, |
| "reward_change_std": 0.16317449882626534, |
| "reward_std": 0.4787884559482336, |
| "rewards/accuracy_reward": 0.1458333395421505, |
| "rewards/cosine_scaled_reward": -0.11097141215577722, |
| "step": 422 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2945.9583740234375, |
| "epoch": 0.48342857142857143, |
| "grad_norm": 0.06767871230840683, |
| "kl": 2.205371856689453e-05, |
| "lambda_div_used": 0.611551009118557, |
| "learning_rate": 1.6508608292777203e-07, |
| "loss": 0.0153, |
| "reward": -0.16482537053525448, |
| "reward_after_mean": -0.16482537053525448, |
| "reward_after_std": 0.5363675616681576, |
| "reward_before_mean": 0.12604539189487696, |
| "reward_before_std": 0.518328445032239, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2908707782626152, |
| "reward_change_min": -0.4775175042450428, |
| "reward_change_std": 0.18889948446303606, |
| "reward_std": 0.5363675802946091, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/cosine_scaled_reward": -0.040621266700327396, |
| "step": 423 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3215.0208740234375, |
| "epoch": 0.4845714285714286, |
| "grad_norm": 0.061716482043266296, |
| "kl": 1.1460855603218079e-05, |
| "lambda_div_used": 0.5937408357858658, |
| "learning_rate": 1.6346804638120098e-07, |
| "loss": 0.0327, |
| "reward": -0.3415182586759329, |
| "reward_after_mean": -0.3415182586759329, |
| "reward_after_std": 0.48476750776171684, |
| "reward_before_mean": -0.08947536488994956, |
| "reward_before_std": 0.4272408355027437, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2520429063588381, |
| "reward_change_min": -0.42084335163235664, |
| "reward_change_std": 0.15120856929570436, |
| "reward_std": 0.4847675133496523, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.17280870315153152, |
| "step": 424 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2154.9375610351562, |
| "epoch": 0.4857142857142857, |
| "grad_norm": 0.09371069073677063, |
| "kl": 3.830809146165848e-05, |
| "lambda_div_used": 0.6315138638019562, |
| "learning_rate": 1.6186884885673413e-07, |
| "loss": 0.0058, |
| "reward": 0.34828917868435383, |
| "reward_after_mean": 0.34828917868435383, |
| "reward_after_std": 0.7491090279072523, |
| "reward_before_mean": 0.91605463065207, |
| "reward_before_std": 0.6105400957167149, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5677654705941677, |
| "reward_change_min": -0.9185826852917671, |
| "reward_change_std": 0.34253316558897495, |
| "reward_std": 0.7491090428084135, |
| "rewards/accuracy_reward": 0.5833333358168602, |
| "rewards/cosine_scaled_reward": 0.3327212668955326, |
| "step": 425 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2205.3125190734863, |
| "epoch": 0.4868571428571429, |
| "grad_norm": 0.09298980236053467, |
| "kl": 2.3175030946731567e-05, |
| "lambda_div_used": 0.6215192526578903, |
| "learning_rate": 1.6028856829700258e-07, |
| "loss": -0.002, |
| "reward": -0.029936233535408974, |
| "reward_after_mean": -0.029936233535408974, |
| "reward_after_std": 0.5604026541113853, |
| "reward_before_mean": 0.3139693345874548, |
| "reward_before_std": 0.5635973755270243, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34390556812286377, |
| "reward_change_min": -0.5907084755599499, |
| "reward_change_std": 0.23498705588281155, |
| "reward_std": 0.5604026671499014, |
| "rewards/accuracy_reward": 0.3125000074505806, |
| "rewards/cosine_scaled_reward": 0.0014693308621644974, |
| "step": 426 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3295.7708740234375, |
| "epoch": 0.488, |
| "grad_norm": 0.0548052154481411, |
| "kl": 2.4527311325073242e-05, |
| "lambda_div_used": 0.5990054532885551, |
| "learning_rate": 1.5872728172265146e-07, |
| "loss": 0.0088, |
| "reward": -0.2071160115301609, |
| "reward_after_mean": -0.2071160115301609, |
| "reward_after_std": 0.49672279693186283, |
| "reward_before_mean": 0.08893106225878, |
| "reward_before_std": 0.454012256115675, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2960470784455538, |
| "reward_change_min": -0.4723533205688, |
| "reward_change_std": 0.18442231137305498, |
| "reward_std": 0.4967228155583143, |
| "rewards/accuracy_reward": 0.18750000558793545, |
| "rewards/cosine_scaled_reward": -0.09856893587857485, |
| "step": 427 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2302.520881652832, |
| "epoch": 0.48914285714285716, |
| "grad_norm": 0.11483462899923325, |
| "kl": 0.000337366946041584, |
| "lambda_div_used": 0.5547136962413788, |
| "learning_rate": 1.5718506522858572e-07, |
| "loss": 0.0311, |
| "reward": -0.42663199454545975, |
| "reward_after_mean": -0.42663199454545975, |
| "reward_after_std": 0.29241302236914635, |
| "reward_before_mean": -0.1547260768711567, |
| "reward_before_std": 0.24498768709599972, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27190591767430305, |
| "reward_change_min": -0.42438486963510513, |
| "reward_change_std": 0.1579811777919531, |
| "reward_std": 0.2924130354076624, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.2380594089627266, |
| "step": 428 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2431.520881652832, |
| "epoch": 0.49028571428571427, |
| "grad_norm": 0.0878632664680481, |
| "kl": 2.4685636162757874e-05, |
| "lambda_div_used": 0.6025609895586967, |
| "learning_rate": 1.5566199398026147e-07, |
| "loss": 0.0378, |
| "reward": -0.057155030546709895, |
| "reward_after_mean": -0.057155030546709895, |
| "reward_after_std": 0.5567535478621721, |
| "reward_before_mean": 0.3208682704716921, |
| "reward_before_std": 0.47341752983629704, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3780232574790716, |
| "reward_change_min": -0.5763550102710724, |
| "reward_change_std": 0.22364642471075058, |
| "reward_std": 0.5567535553127527, |
| "rewards/accuracy_reward": 0.29166666977107525, |
| "rewards/cosine_scaled_reward": 0.029201554832980037, |
| "step": 429 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2577.3125610351562, |
| "epoch": 0.49142857142857144, |
| "grad_norm": 0.08103640377521515, |
| "kl": 1.4763325452804565e-05, |
| "lambda_div_used": 0.6012379750609398, |
| "learning_rate": 1.5415814221002265e-07, |
| "loss": -0.0049, |
| "reward": 0.031602535396814346, |
| "reward_after_mean": 0.031602535396814346, |
| "reward_after_std": 0.5440461356192827, |
| "reward_before_mean": 0.4564003311097622, |
| "reward_before_std": 0.46839726250618696, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.42479780688881874, |
| "reward_change_min": -0.6648109555244446, |
| "reward_change_std": 0.2579617351293564, |
| "reward_std": 0.5440461542457342, |
| "rewards/accuracy_reward": 0.3541666753590107, |
| "rewards/cosine_scaled_reward": 0.1022336557507515, |
| "step": 430 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2690.6458740234375, |
| "epoch": 0.49257142857142855, |
| "grad_norm": 0.06538030505180359, |
| "kl": 2.8170645236968994e-05, |
| "lambda_div_used": 0.5918548330664635, |
| "learning_rate": 1.5267358321348285e-07, |
| "loss": -0.0328, |
| "reward": -0.24143310775980353, |
| "reward_after_mean": -0.24143310775980353, |
| "reward_after_std": 0.481786971911788, |
| "reward_before_mean": 0.05766227189451456, |
| "reward_before_std": 0.41715206764638424, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29909538850188255, |
| "reward_change_min": -0.4450513869524002, |
| "reward_change_std": 0.1692206682637334, |
| "reward_std": 0.4817869979888201, |
| "rewards/accuracy_reward": 0.1458333395421505, |
| "rewards/cosine_scaled_reward": -0.08817105047637597, |
| "step": 431 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2843.750045776367, |
| "epoch": 0.4937142857142857, |
| "grad_norm": 0.05780716612935066, |
| "kl": 3.104284405708313e-05, |
| "lambda_div_used": 0.576793298125267, |
| "learning_rate": 1.5120838934595337e-07, |
| "loss": 0.0053, |
| "reward": -0.42155745439231396, |
| "reward_after_mean": -0.42155745439231396, |
| "reward_after_std": 0.4230448566377163, |
| "reward_before_mean": -0.19034346495755017, |
| "reward_before_std": 0.35024640895426273, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.23121398873627186, |
| "reward_change_min": -0.3715337961912155, |
| "reward_change_std": 0.1338915005326271, |
| "reward_std": 0.4230448678135872, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.2528434665873647, |
| "step": 432 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2992.062530517578, |
| "epoch": 0.4948571428571429, |
| "grad_norm": 0.06562553346157074, |
| "kl": 3.0972063541412354e-05, |
| "lambda_div_used": 0.6079665347933769, |
| "learning_rate": 1.4976263201891613e-07, |
| "loss": 0.1201, |
| "reward": -0.26266857516020536, |
| "reward_after_mean": -0.26266857516020536, |
| "reward_after_std": 0.5306479204446077, |
| "reward_before_mean": -0.0102414321154356, |
| "reward_before_std": 0.5030816271901131, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25242713652551174, |
| "reward_change_min": -0.4955142140388489, |
| "reward_change_std": 0.1781205264851451, |
| "reward_std": 0.5306479260325432, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.15607476886361837, |
| "step": 433 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2915.625030517578, |
| "epoch": 0.496, |
| "grad_norm": 0.06934478878974915, |
| "kl": 3.8422178477048874e-05, |
| "lambda_div_used": 0.5604980885982513, |
| "learning_rate": 1.483363816965435e-07, |
| "loss": -0.004, |
| "reward": -0.4280230412259698, |
| "reward_after_mean": -0.4280230412259698, |
| "reward_after_std": 0.32627478428184986, |
| "reward_before_mean": -0.15262745507061481, |
| "reward_before_std": 0.2731653768569231, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27539557591080666, |
| "reward_change_min": -0.44822341948747635, |
| "reward_change_std": 0.16339552495628595, |
| "reward_std": 0.32627478800714016, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.1942941201850772, |
| "step": 434 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2320.750026702881, |
| "epoch": 0.49714285714285716, |
| "grad_norm": 0.13675029575824738, |
| "kl": 4.8140063881874084e-05, |
| "lambda_div_used": 0.5811168253421783, |
| "learning_rate": 1.469297078922642e-07, |
| "loss": -0.0374, |
| "reward": -0.17623315937817097, |
| "reward_after_mean": -0.17623315937817097, |
| "reward_after_std": 0.483237961307168, |
| "reward_before_mean": 0.19585114251822233, |
| "reward_before_std": 0.3759047882631421, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.37208431400358677, |
| "reward_change_min": -0.5468379594385624, |
| "reward_change_std": 0.20962903555482626, |
| "reward_std": 0.4832379762083292, |
| "rewards/accuracy_reward": 0.20833333395421505, |
| "rewards/cosine_scaled_reward": -0.01248218398541212, |
| "step": 435 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2316.3125381469727, |
| "epoch": 0.4982857142857143, |
| "grad_norm": 0.09783437103033066, |
| "kl": 2.839416265487671e-05, |
| "lambda_div_used": 0.6289880573749542, |
| "learning_rate": 1.4554267916537495e-07, |
| "loss": 0.0826, |
| "reward": 0.12041758373379707, |
| "reward_after_mean": 0.12041758373379707, |
| "reward_after_std": 0.6426001656800508, |
| "reward_before_mean": 0.5348608233034611, |
| "reward_before_std": 0.6021685730665922, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.41444322280585766, |
| "reward_change_min": -0.6993351243436337, |
| "reward_change_std": 0.271469890139997, |
| "reward_std": 0.6426001694053411, |
| "rewards/accuracy_reward": 0.3958333395421505, |
| "rewards/cosine_scaled_reward": 0.13902746886014938, |
| "step": 436 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2954.0416717529297, |
| "epoch": 0.49942857142857144, |
| "grad_norm": 0.05496148392558098, |
| "kl": 0.00023943744599819183, |
| "lambda_div_used": 0.5626013651490211, |
| "learning_rate": 1.4417536311769885e-07, |
| "loss": 0.05, |
| "reward": -0.42826351523399353, |
| "reward_after_mean": -0.42826351523399353, |
| "reward_after_std": 0.33650987036526203, |
| "reward_before_mean": -0.16868688352406025, |
| "reward_before_std": 0.2832178361713886, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25957662612199783, |
| "reward_change_min": -0.42033713683485985, |
| "reward_change_std": 0.1560918828472495, |
| "reward_std": 0.3365098759531975, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.2311868779361248, |
| "step": 437 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3007.500015258789, |
| "epoch": 0.5005714285714286, |
| "grad_norm": 0.05682806298136711, |
| "kl": 4.358415026217699e-06, |
| "lambda_div_used": 0.6039799600839615, |
| "learning_rate": 1.4282782639029128e-07, |
| "loss": 0.0289, |
| "reward": -0.29300207551568747, |
| "reward_after_mean": -0.29300207551568747, |
| "reward_after_std": 0.5204536523669958, |
| "reward_before_mean": -0.03094739466905594, |
| "reward_before_std": 0.4829921592026949, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2620546855032444, |
| "reward_change_min": -0.5145607963204384, |
| "reward_change_std": 0.18371508549898863, |
| "reward_std": 0.5204536579549313, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.15594739385414869, |
| "step": 438 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2249.5000076293945, |
| "epoch": 0.5017142857142857, |
| "grad_norm": 0.0828532725572586, |
| "kl": 3.217160701751709e-05, |
| "lambda_div_used": 0.5708463862538338, |
| "learning_rate": 1.4150013466019114e-07, |
| "loss": 0.021, |
| "reward": -0.40206424333155155, |
| "reward_after_mean": -0.40206424333155155, |
| "reward_after_std": 0.40689974650740623, |
| "reward_before_mean": -0.14063972979784012, |
| "reward_before_std": 0.32079045102000237, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.261424507945776, |
| "reward_change_min": -0.38026969507336617, |
| "reward_change_std": 0.14067976083606482, |
| "reward_std": 0.406899768859148, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.1823064020718448, |
| "step": 439 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2964.000011444092, |
| "epoch": 0.5028571428571429, |
| "grad_norm": 0.07350306212902069, |
| "kl": 2.7701258659362793e-05, |
| "lambda_div_used": 0.5365751013159752, |
| "learning_rate": 1.4019235263722034e-07, |
| "loss": 0.0201, |
| "reward": -0.5367953963577747, |
| "reward_after_mean": -0.5367953963577747, |
| "reward_after_std": 0.231884878128767, |
| "reward_before_mean": -0.28081973269581795, |
| "reward_before_std": 0.16059848852455616, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25597566179931164, |
| "reward_change_min": -0.38940757140517235, |
| "reward_change_std": 0.13862022012472153, |
| "reward_std": 0.2318848893046379, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.28081973642110825, |
| "step": 440 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3007.770896911621, |
| "epoch": 0.504, |
| "grad_norm": 0.07498018443584442, |
| "kl": 2.944841980934143e-05, |
| "lambda_div_used": 0.6011143997311592, |
| "learning_rate": 1.3890454406082956e-07, |
| "loss": -0.0116, |
| "reward": -0.13889071717858315, |
| "reward_after_mean": -0.13889071717858315, |
| "reward_after_std": 0.4864419028162956, |
| "reward_before_mean": 0.17084867507219315, |
| "reward_before_std": 0.4689741334877908, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3097394183278084, |
| "reward_change_min": -0.514279019087553, |
| "reward_change_std": 0.20097953081130981, |
| "reward_std": 0.48644191212952137, |
| "rewards/accuracy_reward": 0.2291666753590107, |
| "rewards/cosine_scaled_reward": -0.05831799004226923, |
| "step": 441 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2837.4583435058594, |
| "epoch": 0.5051428571428571, |
| "grad_norm": 0.0673719272017479, |
| "kl": 3.160163760185242e-05, |
| "lambda_div_used": 0.5689455196261406, |
| "learning_rate": 1.3763677169699217e-07, |
| "loss": 0.0084, |
| "reward": -0.31079866108484566, |
| "reward_after_mean": -0.31079866108484566, |
| "reward_after_std": 0.3982803635299206, |
| "reward_before_mean": -0.010625829687342048, |
| "reward_before_std": 0.31661996035836637, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.30017283000051975, |
| "reward_change_min": -0.4261031448841095, |
| "reward_change_std": 0.16800945159047842, |
| "reward_std": 0.3982803765684366, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.1147925069089979, |
| "step": 442 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3194.270854949951, |
| "epoch": 0.5062857142857143, |
| "grad_norm": 0.05984271690249443, |
| "kl": 1.745857298374176e-05, |
| "lambda_div_used": 0.5607094466686249, |
| "learning_rate": 1.3638909733514452e-07, |
| "loss": 0.0053, |
| "reward": -0.38279488682746887, |
| "reward_after_mean": -0.38279488682746887, |
| "reward_after_std": 0.33906700275838375, |
| "reward_before_mean": -0.09817820321768522, |
| "reward_before_std": 0.2764374865218997, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2846166864037514, |
| "reward_change_min": -0.4438001811504364, |
| "reward_change_std": 0.16594033408910036, |
| "reward_std": 0.3390670083463192, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.16067820694297552, |
| "step": 443 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2902.2291717529297, |
| "epoch": 0.5074285714285715, |
| "grad_norm": 0.0638304203748703, |
| "kl": 1.5033408999443054e-05, |
| "lambda_div_used": 0.5774018093943596, |
| "learning_rate": 1.351615817851748e-07, |
| "loss": -0.0355, |
| "reward": -0.3352237604558468, |
| "reward_after_mean": -0.3352237604558468, |
| "reward_after_std": 0.4039857666939497, |
| "reward_before_mean": -0.05285666696727276, |
| "reward_before_std": 0.35096561443060637, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2823671065270901, |
| "reward_change_min": -0.42594215646386147, |
| "reward_change_std": 0.16656992863863707, |
| "reward_std": 0.40398577041924, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.15702332742512226, |
| "step": 444 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3006.916717529297, |
| "epoch": 0.5085714285714286, |
| "grad_norm": 0.06397882103919983, |
| "kl": 2.842303365468979e-05, |
| "lambda_div_used": 0.6194909662008286, |
| "learning_rate": 1.3395428487445914e-07, |
| "loss": 0.0715, |
| "reward": -0.05263347551226616, |
| "reward_after_mean": -0.05263347551226616, |
| "reward_after_std": 0.5679135732352734, |
| "reward_before_mean": 0.2764017879962921, |
| "reward_before_std": 0.5510673765093088, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3290352523326874, |
| "reward_change_min": -0.5197134874761105, |
| "reward_change_std": 0.21186745446175337, |
| "reward_std": 0.5679135844111443, |
| "rewards/accuracy_reward": 0.27083334140479565, |
| "rewards/cosine_scaled_reward": 0.0055684298276901245, |
| "step": 445 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3028.8333892822266, |
| "epoch": 0.5097142857142857, |
| "grad_norm": 0.06140498071908951, |
| "kl": 3.2648444175720215e-05, |
| "lambda_div_used": 0.606703408062458, |
| "learning_rate": 1.3276726544494571e-07, |
| "loss": -0.0241, |
| "reward": -0.2073356769979, |
| "reward_after_mean": -0.2073356769979, |
| "reward_after_std": 0.5240648984909058, |
| "reward_before_mean": 0.0723939798772335, |
| "reward_before_std": 0.4952498711645603, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27972964756190777, |
| "reward_change_min": -0.5379137881100178, |
| "reward_change_std": 0.1909602265805006, |
| "reward_std": 0.5240649078041315, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.07343936339020729, |
| "step": 446 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2571.166721343994, |
| "epoch": 0.5108571428571429, |
| "grad_norm": 0.08778514713048935, |
| "kl": 9.072478860616684e-06, |
| "lambda_div_used": 0.5970501974225044, |
| "learning_rate": 1.316005813502869e-07, |
| "loss": 0.0018, |
| "reward": -0.19514422863721848, |
| "reward_after_mean": -0.19514422863721848, |
| "reward_after_std": 0.4898754768073559, |
| "reward_before_mean": 0.10967571474611759, |
| "reward_before_std": 0.4461768325418234, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3048199340701103, |
| "reward_change_min": -0.48957687243819237, |
| "reward_change_std": 0.18721044715493917, |
| "reward_std": 0.4898754861205816, |
| "rewards/accuracy_reward": 0.18750000558793545, |
| "rewards/cosine_scaled_reward": -0.07782429456710815, |
| "step": 447 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2039.6042022705078, |
| "epoch": 0.512, |
| "grad_norm": 0.12122303992509842, |
| "kl": 3.530830144882202e-05, |
| "lambda_div_used": 0.5591797679662704, |
| "learning_rate": 1.3045428945301953e-07, |
| "loss": 0.0057, |
| "reward": -0.3564775697886944, |
| "reward_after_mean": -0.3564775697886944, |
| "reward_after_std": 0.33722602762281895, |
| "reward_before_mean": -0.06660962291061878, |
| "reward_before_std": 0.27086107339709997, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.28986795619130135, |
| "reward_change_min": -0.4345347508788109, |
| "reward_change_std": 0.1656003799289465, |
| "reward_std": 0.33722603134810925, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.14994295500218868, |
| "step": 448 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2680.0208435058594, |
| "epoch": 0.5131428571428571, |
| "grad_norm": 0.07745695859193802, |
| "kl": 3.194063901901245e-05, |
| "lambda_div_used": 0.5827131420373917, |
| "learning_rate": 1.2932844562179352e-07, |
| "loss": -0.0247, |
| "reward": -0.278532937169075, |
| "reward_after_mean": -0.278532937169075, |
| "reward_after_std": 0.44566163793206215, |
| "reward_before_mean": 0.01611769199371338, |
| "reward_before_std": 0.3775772461667657, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29465061984956264, |
| "reward_change_min": -0.4545559212565422, |
| "reward_change_std": 0.17489958554506302, |
| "reward_std": 0.44566163793206215, |
| "rewards/accuracy_reward": 0.18750000558793545, |
| "rewards/cosine_scaled_reward": -0.17138232477009296, |
| "step": 449 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2405.0833702087402, |
| "epoch": 0.5142857142857142, |
| "grad_norm": 0.07359456270933151, |
| "kl": 4.271417856216431e-05, |
| "lambda_div_used": 0.5313882529735565, |
| "learning_rate": 1.2822310472864885e-07, |
| "loss": 0.0047, |
| "reward": -0.40455422177910805, |
| "reward_after_mean": -0.40455422177910805, |
| "reward_after_std": 0.27055008336901665, |
| "reward_before_mean": -0.05470774322748184, |
| "reward_before_std": 0.13779542688280344, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34984651021659374, |
| "reward_change_min": -0.482446551322937, |
| "reward_change_std": 0.18013106007128954, |
| "reward_std": 0.2705500964075327, |
| "rewards/accuracy_reward": 0.125, |
| "rewards/cosine_scaled_reward": -0.17970772832632065, |
| "step": 450 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2726.7916870117188, |
| "epoch": 0.5154285714285715, |
| "grad_norm": 0.10456772893667221, |
| "kl": 5.123019218444824e-05, |
| "lambda_div_used": 0.5659012496471405, |
| "learning_rate": 1.2713832064634125e-07, |
| "loss": 0.0437, |
| "reward": -0.233762688934803, |
| "reward_after_mean": -0.233762688934803, |
| "reward_after_std": 0.3937326893210411, |
| "reward_before_mean": 0.14449233608320355, |
| "reward_before_std": 0.30359079129993916, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3782550562173128, |
| "reward_change_min": -0.5748258531093597, |
| "reward_change_std": 0.21745321340858936, |
| "reward_std": 0.3937326930463314, |
| "rewards/accuracy_reward": 0.1875, |
| "rewards/cosine_scaled_reward": -0.0430076620541513, |
| "step": 451 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3173.812530517578, |
| "epoch": 0.5165714285714286, |
| "grad_norm": 0.05382152646780014, |
| "kl": 1.621060073375702e-05, |
| "lambda_div_used": 0.628274716436863, |
| "learning_rate": 1.260741462457165e-07, |
| "loss": 0.0152, |
| "reward": 0.08829959481954575, |
| "reward_after_mean": 0.08829959481954575, |
| "reward_after_std": 0.6491915434598923, |
| "reward_before_mean": 0.49067158019170165, |
| "reward_before_std": 0.5948440972715616, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4023719858378172, |
| "reward_change_min": -0.6429549157619476, |
| "reward_change_std": 0.2526876190677285, |
| "reward_std": 0.6491915658116341, |
| "rewards/accuracy_reward": 0.37500000558793545, |
| "rewards/cosine_scaled_reward": 0.11567158252000809, |
| "step": 452 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2601.1666870117188, |
| "epoch": 0.5177142857142857, |
| "grad_norm": 0.10710335522890091, |
| "kl": 4.57763671875e-05, |
| "lambda_div_used": 0.5769856572151184, |
| "learning_rate": 1.2503063339313356e-07, |
| "loss": -0.0233, |
| "reward": -0.4120060931891203, |
| "reward_after_mean": -0.4120060931891203, |
| "reward_after_std": 0.4247955661267042, |
| "reward_before_mean": -0.17395742796361446, |
| "reward_before_std": 0.3489131908863783, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.23804865032434464, |
| "reward_change_min": -0.3744778670370579, |
| "reward_change_std": 0.12989939749240875, |
| "reward_std": 0.4247955847531557, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.21562409959733486, |
| "step": 453 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2794.8958740234375, |
| "epoch": 0.5188571428571429, |
| "grad_norm": 0.059399981051683426, |
| "kl": 1.829490065574646e-05, |
| "lambda_div_used": 0.604412779211998, |
| "learning_rate": 1.2400783294793668e-07, |
| "loss": 0.0084, |
| "reward": -0.23108407109975815, |
| "reward_after_mean": -0.23108407109975815, |
| "reward_after_std": 0.5207228269428015, |
| "reward_before_mean": 0.05918463226407766, |
| "reward_before_std": 0.4853199487552047, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29026869870722294, |
| "reward_change_min": -0.5276229903101921, |
| "reward_change_std": 0.1960765514522791, |
| "reward_std": 0.5207228306680918, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.08664871653309092, |
| "step": 454 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3120.0416717529297, |
| "epoch": 0.52, |
| "grad_norm": 0.07823975384235382, |
| "kl": 1.1070631444454193e-05, |
| "lambda_div_used": 0.5381916239857674, |
| "learning_rate": 1.2300579475997657e-07, |
| "loss": 0.0035, |
| "reward": -0.5376988351345062, |
| "reward_after_mean": -0.5376988351345062, |
| "reward_after_std": 0.2399882897734642, |
| "reward_before_mean": -0.28268107399344444, |
| "reward_before_std": 0.16770172398537397, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25501775927841663, |
| "reward_change_min": -0.38403724879026413, |
| "reward_change_std": 0.1382020702585578, |
| "reward_std": 0.23998829536139965, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.28268107026815414, |
| "step": 455 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3183.125, |
| "epoch": 0.5211428571428571, |
| "grad_norm": 0.07441714406013489, |
| "kl": 1.7248094081878662e-05, |
| "lambda_div_used": 0.535825714468956, |
| "learning_rate": 1.220245676671809e-07, |
| "loss": -0.0061, |
| "reward": -0.5242688432335854, |
| "reward_after_mean": -0.5242688432335854, |
| "reward_after_std": 0.2355129700154066, |
| "reward_before_mean": -0.2603513076901436, |
| "reward_before_std": 0.15733365854248405, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2639175299555063, |
| "reward_change_min": -0.3936190530657768, |
| "reward_change_std": 0.14126356784254313, |
| "reward_std": 0.23551297932863235, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2603513114154339, |
| "step": 456 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3090.9166717529297, |
| "epoch": 0.5222857142857142, |
| "grad_norm": 0.06740739941596985, |
| "kl": 1.6771256923675537e-05, |
| "lambda_div_used": 0.5661213397979736, |
| "learning_rate": 1.2106419949317388e-07, |
| "loss": 0.0183, |
| "reward": -0.2666686773300171, |
| "reward_after_mean": -0.2666686773300171, |
| "reward_after_std": 0.3630063198506832, |
| "reward_before_mean": 0.06939902156591415, |
| "reward_before_std": 0.3003841144964099, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3360677044838667, |
| "reward_change_min": -0.5109980814158916, |
| "reward_change_std": 0.19570716377347708, |
| "reward_std": 0.36300632916390896, |
| "rewards/accuracy_reward": 0.1458333395421505, |
| "rewards/cosine_scaled_reward": -0.0764343123883009, |
| "step": 457 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2337.0416717529297, |
| "epoch": 0.5234285714285715, |
| "grad_norm": 0.10038017481565475, |
| "kl": 1.5580561012029648e-05, |
| "lambda_div_used": 0.587464913725853, |
| "learning_rate": 1.2012473704494537e-07, |
| "loss": 0.0337, |
| "reward": -0.2921891398727894, |
| "reward_after_mean": -0.2921891398727894, |
| "reward_after_std": 0.46863692067563534, |
| "reward_before_mean": -0.017078701872378588, |
| "reward_before_std": 0.3978575337678194, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27511043660342693, |
| "reward_change_min": -0.4165792725980282, |
| "reward_change_std": 0.15801922790706158, |
| "reward_std": 0.4686369299888611, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.14207870024256408, |
| "step": 458 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1897.9791870117188, |
| "epoch": 0.5245714285714286, |
| "grad_norm": 0.10566994547843933, |
| "kl": 3.1791627407073975e-05, |
| "lambda_div_used": 0.6483638733625412, |
| "learning_rate": 1.1920622611056974e-07, |
| "loss": -0.0327, |
| "reward": 0.04763873480260372, |
| "reward_after_mean": 0.04763873480260372, |
| "reward_after_std": 0.6893943976610899, |
| "reward_before_mean": 0.37104589492082596, |
| "reward_before_std": 0.6905025038868189, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3234071359038353, |
| "reward_change_min": -0.573150984942913, |
| "reward_change_std": 0.22597133833914995, |
| "reward_std": 0.6893944274634123, |
| "rewards/accuracy_reward": 0.29166667349636555, |
| "rewards/cosine_scaled_reward": 0.07937921397387981, |
| "step": 459 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3152.9583587646484, |
| "epoch": 0.5257142857142857, |
| "grad_norm": 0.0553775392472744, |
| "kl": 1.6976147890090942e-05, |
| "lambda_div_used": 0.6212490126490593, |
| "learning_rate": 1.1830871145697412e-07, |
| "loss": -0.0403, |
| "reward": -0.2310712798498571, |
| "reward_after_mean": -0.2310712798498571, |
| "reward_after_std": 0.5967046339064837, |
| "reward_before_mean": 0.018506756518036127, |
| "reward_before_std": 0.5584379080682993, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24957803264260292, |
| "reward_change_min": -0.45073715783655643, |
| "reward_change_std": 0.16405375488102436, |
| "reward_std": 0.5967046469449997, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.1273265864001587, |
| "step": 460 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2965.625015258789, |
| "epoch": 0.5268571428571428, |
| "grad_norm": 0.0683179497718811, |
| "kl": 3.898143768310547e-05, |
| "lambda_div_used": 0.5838751494884491, |
| "learning_rate": 1.1743223682775649e-07, |
| "loss": 0.0201, |
| "reward": -0.36440329253673553, |
| "reward_after_mean": -0.36440329253673553, |
| "reward_after_std": 0.44049660488963127, |
| "reward_before_mean": -0.10854836623184383, |
| "reward_before_std": 0.3863721750676632, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25585492700338364, |
| "reward_change_min": -0.41783962957561016, |
| "reward_change_std": 0.15460436698049307, |
| "reward_std": 0.44049660861492157, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.19188169576227665, |
| "step": 461 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2874.3125076293945, |
| "epoch": 0.528, |
| "grad_norm": 0.07832667976617813, |
| "kl": 2.7470290660858154e-05, |
| "lambda_div_used": 0.5419389680027962, |
| "learning_rate": 1.1657684494105386e-07, |
| "loss": -0.0381, |
| "reward": -0.5123195722699165, |
| "reward_after_mean": -0.5123195722699165, |
| "reward_after_std": 0.2568210382014513, |
| "reward_before_mean": -0.2502432279288769, |
| "reward_before_std": 0.1842196974903345, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2620763499289751, |
| "reward_change_min": -0.4005816727876663, |
| "reward_change_std": 0.14368234388530254, |
| "reward_std": 0.2568210456520319, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2502432242035866, |
| "step": 462 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2986.8333740234375, |
| "epoch": 0.5291428571428571, |
| "grad_norm": 0.053950026631355286, |
| "kl": 8.771196007728577e-06, |
| "lambda_div_used": 0.622698076069355, |
| "learning_rate": 1.1574257748745986e-07, |
| "loss": 0.0106, |
| "reward": -0.2041953857988119, |
| "reward_after_mean": -0.2041953857988119, |
| "reward_after_std": 0.6069091446697712, |
| "reward_before_mean": 0.05050618201494217, |
| "reward_before_std": 0.5684067364782095, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25470156222581863, |
| "reward_change_min": -0.4568561315536499, |
| "reward_change_std": 0.16765617858618498, |
| "reward_std": 0.6069091446697712, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/cosine_scaled_reward": -0.09532715613022447, |
| "step": 463 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1749.0416717529297, |
| "epoch": 0.5302857142857142, |
| "grad_norm": 0.11516103893518448, |
| "kl": 3.466010093688965e-05, |
| "lambda_div_used": 0.5341480076313019, |
| "learning_rate": 1.1492947512799328e-07, |
| "loss": -0.0088, |
| "reward": -0.040389321744441986, |
| "reward_after_mean": -0.040389321744441986, |
| "reward_after_std": 0.37696985714137554, |
| "reward_before_mean": 0.5386080192402005, |
| "reward_before_std": 0.14989514648914337, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.5789973307400942, |
| "reward_change_min": -0.7674467526376247, |
| "reward_change_std": 0.2917799688875675, |
| "reward_std": 0.37696986459195614, |
| "rewards/accuracy_reward": 0.375, |
| "rewards/cosine_scaled_reward": 0.16360801365226507, |
| "step": 464 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3036.854179382324, |
| "epoch": 0.5314285714285715, |
| "grad_norm": 0.08283663541078568, |
| "kl": 2.4955719709396362e-05, |
| "lambda_div_used": 0.572467640042305, |
| "learning_rate": 1.1413757749211602e-07, |
| "loss": 0.0373, |
| "reward": -0.40219551732297987, |
| "reward_after_mean": -0.40219551732297987, |
| "reward_after_std": 0.40797613374888897, |
| "reward_before_mean": -0.1455877646803856, |
| "reward_before_std": 0.32708599977195263, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.256607748568058, |
| "reward_change_min": -0.37336331233382225, |
| "reward_change_std": 0.1377662243321538, |
| "reward_std": 0.40797614119946957, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.18725443072617054, |
| "step": 465 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3122.9583435058594, |
| "epoch": 0.5325714285714286, |
| "grad_norm": 0.07085831463336945, |
| "kl": 2.0582228899002075e-05, |
| "lambda_div_used": 0.5319224968552589, |
| "learning_rate": 1.1336692317580158e-07, |
| "loss": -0.0563, |
| "reward": -0.36368466913700104, |
| "reward_after_mean": -0.36368466913700104, |
| "reward_after_std": 0.27606455981731415, |
| "reward_before_mean": 0.005439521744847298, |
| "reward_before_std": 0.14014938473701477, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.36912417598068714, |
| "reward_change_min": -0.5061142668128014, |
| "reward_change_std": 0.1899967910721898, |
| "reward_std": 0.2760645691305399, |
| "rewards/accuracy_reward": 0.125, |
| "rewards/cosine_scaled_reward": -0.11956048291176558, |
| "step": 466 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3113.2708854675293, |
| "epoch": 0.5337142857142857, |
| "grad_norm": 0.07966674119234085, |
| "kl": 3.802310675382614e-05, |
| "lambda_div_used": 0.5612561926245689, |
| "learning_rate": 1.1261754973965422e-07, |
| "loss": -0.0067, |
| "reward": -0.3137255348265171, |
| "reward_after_mean": -0.3137255348265171, |
| "reward_after_std": 0.39207486622035503, |
| "reward_before_mean": 0.03320633992552757, |
| "reward_before_std": 0.2764817178249359, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3469318952411413, |
| "reward_change_min": -0.4828169047832489, |
| "reward_change_std": 0.18068420328199863, |
| "reward_std": 0.3920748811215162, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/cosine_scaled_reward": -0.11262698657810688, |
| "step": 467 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2898.770854949951, |
| "epoch": 0.5348571428571428, |
| "grad_norm": 0.07577298581600189, |
| "kl": 4.957616329193115e-05, |
| "lambda_div_used": 0.5363663136959076, |
| "learning_rate": 1.1188949370707787e-07, |
| "loss": 0.0338, |
| "reward": -0.5352834053337574, |
| "reward_after_mean": -0.5352834053337574, |
| "reward_after_std": 0.22454985231161118, |
| "reward_before_mean": -0.2775777019560337, |
| "reward_before_std": 0.15978037798777223, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2577057033777237, |
| "reward_change_min": -0.3789172098040581, |
| "reward_change_std": 0.14186344109475613, |
| "reward_std": 0.22454985417425632, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.27757770381867886, |
| "step": 468 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2854.8750228881836, |
| "epoch": 0.536, |
| "grad_norm": 0.09182782471179962, |
| "kl": 2.0341947674751282e-05, |
| "lambda_div_used": 0.5550094842910767, |
| "learning_rate": 1.1118279056249653e-07, |
| "loss": 0.0301, |
| "reward": -0.35969678312540054, |
| "reward_after_mean": -0.35969678312540054, |
| "reward_after_std": 0.30523936823010445, |
| "reward_before_mean": -0.04356633126735687, |
| "reward_before_std": 0.2476841462776065, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.31613045185804367, |
| "reward_change_min": -0.45744797959923744, |
| "reward_change_std": 0.17962745483964682, |
| "reward_std": 0.3052393738180399, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.12689966335892677, |
| "step": 469 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3333.562530517578, |
| "epoch": 0.5371428571428571, |
| "grad_norm": 0.05610484257340431, |
| "kl": 2.9811635613441467e-05, |
| "lambda_div_used": 0.5911690816283226, |
| "learning_rate": 1.1049747474962444e-07, |
| "loss": 0.0307, |
| "reward": -0.37843912467360497, |
| "reward_after_mean": -0.37843912467360497, |
| "reward_after_std": 0.49228920973837376, |
| "reward_before_mean": -0.14763083557772916, |
| "reward_before_std": 0.4128831513226032, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.230808287858963, |
| "reward_change_min": -0.3457215949892998, |
| "reward_change_std": 0.12627399526536465, |
| "reward_std": 0.49228921718895435, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.21013083728030324, |
| "step": 470 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3032.312515258789, |
| "epoch": 0.5382857142857143, |
| "grad_norm": 0.053547170013189316, |
| "kl": 2.0138919353485107e-05, |
| "lambda_div_used": 0.5520770102739334, |
| "learning_rate": 1.0983357966978745e-07, |
| "loss": -0.0036, |
| "reward": -0.32707202807068825, |
| "reward_after_mean": -0.32707202807068825, |
| "reward_after_std": 0.30774626694619656, |
| "reward_before_mean": 0.006061417981982231, |
| "reward_before_std": 0.23228682670742273, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3331334535032511, |
| "reward_change_min": -0.47613633051514626, |
| "reward_change_std": 0.181988756172359, |
| "reward_std": 0.3077462762594223, |
| "rewards/accuracy_reward": 0.1041666716337204, |
| "rewards/cosine_scaled_reward": -0.09810524806380272, |
| "step": 471 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3115.8333435058594, |
| "epoch": 0.5394285714285715, |
| "grad_norm": 0.05183921754360199, |
| "kl": 1.757591962814331e-05, |
| "lambda_div_used": 0.6081600487232208, |
| "learning_rate": 1.0919113768029517e-07, |
| "loss": 0.0248, |
| "reward": -0.22974545462056994, |
| "reward_after_mean": -0.22974545462056994, |
| "reward_after_std": 0.5618578754365444, |
| "reward_before_mean": 0.03380160592496395, |
| "reward_before_std": 0.4960998175665736, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2635470647364855, |
| "reward_change_min": -0.4026007801294327, |
| "reward_change_std": 0.1525269951671362, |
| "reward_std": 0.5618579015135765, |
| "rewards/accuracy_reward": 0.14583333767950535, |
| "rewards/cosine_scaled_reward": -0.1120317269815132, |
| "step": 472 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3389.7291870117188, |
| "epoch": 0.5405714285714286, |
| "grad_norm": 0.05399390682578087, |
| "kl": 2.3730099201202393e-05, |
| "lambda_div_used": 0.6023471653461456, |
| "learning_rate": 1.0857018009286381e-07, |
| "loss": -0.0081, |
| "reward": -0.16527403378859162, |
| "reward_after_mean": -0.16527403378859162, |
| "reward_after_std": 0.4859522972255945, |
| "reward_before_mean": 0.15660450607538223, |
| "reward_before_std": 0.469901567324996, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3218785412609577, |
| "reward_change_min": -0.5562809407711029, |
| "reward_change_std": 0.21282578073441982, |
| "reward_std": 0.48595230281352997, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/cosine_scaled_reward": -0.051728841848671436, |
| "step": 473 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2609.3541946411133, |
| "epoch": 0.5417142857142857, |
| "grad_norm": 0.07730484753847122, |
| "kl": 1.3811513781547546e-05, |
| "lambda_div_used": 0.5951687097549438, |
| "learning_rate": 1.0797073717209013e-07, |
| "loss": 0.085, |
| "reward": 0.0976928174495697, |
| "reward_after_mean": 0.0976928174495697, |
| "reward_after_std": 0.5153173375874758, |
| "reward_before_mean": 0.5706379320472479, |
| "reward_before_std": 0.4373670984059572, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.47294510155916214, |
| "reward_change_min": -0.6950589865446091, |
| "reward_change_std": 0.27640796452760696, |
| "reward_std": 0.515317365527153, |
| "rewards/accuracy_reward": 0.3958333432674408, |
| "rewards/cosine_scaled_reward": 0.1748045664280653, |
| "step": 474 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2361.020851135254, |
| "epoch": 0.5428571428571428, |
| "grad_norm": 0.08214546740055084, |
| "kl": 9.525567293167114e-06, |
| "lambda_div_used": 0.6207183972001076, |
| "learning_rate": 1.0739283813397639e-07, |
| "loss": 0.0204, |
| "reward": 0.0007177861407399178, |
| "reward_after_mean": 0.0007177861407399178, |
| "reward_after_std": 0.5817896965891123, |
| "reward_before_mean": 0.34289272502064705, |
| "reward_before_std": 0.5596933793276548, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3421749472618103, |
| "reward_change_min": -0.5431834943592548, |
| "reward_change_std": 0.21874003671109676, |
| "reward_std": 0.581789730116725, |
| "rewards/accuracy_reward": 0.29166667722165585, |
| "rewards/cosine_scaled_reward": 0.05122605012729764, |
| "step": 475 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2583.8333740234375, |
| "epoch": 0.544, |
| "grad_norm": 0.08052952587604523, |
| "kl": 3.0007213354110718e-05, |
| "lambda_div_used": 0.6784488782286644, |
| "learning_rate": 1.068365111445064e-07, |
| "loss": 0.0154, |
| "reward": -0.031517775263637304, |
| "reward_after_mean": -0.031517775263637304, |
| "reward_after_std": 0.8641770519316196, |
| "reward_before_mean": 0.2038349723443389, |
| "reward_before_std": 0.8286011293530464, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.23535274900496006, |
| "reward_change_min": -0.41804002970457077, |
| "reward_change_std": 0.16118939872831106, |
| "reward_std": 0.8641770891845226, |
| "rewards/accuracy_reward": 0.22916666977107525, |
| "rewards/cosine_scaled_reward": -0.025331700686365366, |
| "step": 476 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2308.291732788086, |
| "epoch": 0.5451428571428572, |
| "grad_norm": 0.06839341670274734, |
| "kl": 8.73953104019165e-06, |
| "lambda_div_used": 0.6096622571349144, |
| "learning_rate": 1.063017833182728e-07, |
| "loss": 0.1111, |
| "reward": 0.19456686079502106, |
| "reward_after_mean": 0.19456686079502106, |
| "reward_after_std": 0.5847878716886044, |
| "reward_before_mean": 0.678333050571382, |
| "reward_before_std": 0.5006472393870354, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.48376619443297386, |
| "reward_change_min": -0.6835142783820629, |
| "reward_change_std": 0.2774948738515377, |
| "reward_std": 0.5847878959029913, |
| "rewards/accuracy_reward": 0.45833334885537624, |
| "rewards/cosine_scaled_reward": 0.21999971382319927, |
| "step": 477 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3169.3125610351562, |
| "epoch": 0.5462857142857143, |
| "grad_norm": 0.052864328026771545, |
| "kl": 6.990041583776474e-06, |
| "lambda_div_used": 0.5758631080389023, |
| "learning_rate": 1.0578868071715544e-07, |
| "loss": 0.0349, |
| "reward": -0.23655124567449093, |
| "reward_after_mean": -0.23655124567449093, |
| "reward_after_std": 0.4532997701317072, |
| "reward_before_mean": 0.11144567281007767, |
| "reward_before_std": 0.3459607223048806, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3479969333857298, |
| "reward_change_min": -0.5187661461532116, |
| "reward_change_std": 0.19546091184020042, |
| "reward_std": 0.4532997887581587, |
| "rewards/accuracy_reward": 0.20833333395421505, |
| "rewards/cosine_scaled_reward": -0.09688764810562134, |
| "step": 478 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2825.916702270508, |
| "epoch": 0.5474285714285714, |
| "grad_norm": 0.08355361968278885, |
| "kl": 3.739655949175358e-05, |
| "lambda_div_used": 0.5902442187070847, |
| "learning_rate": 1.0529722834905125e-07, |
| "loss": 0.033, |
| "reward": -0.37585191056132317, |
| "reward_after_mean": -0.37585191056132317, |
| "reward_after_std": 0.45466959848999977, |
| "reward_before_mean": -0.12638765759766102, |
| "reward_before_std": 0.4168459586799145, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2494642548263073, |
| "reward_change_min": -0.4229847304522991, |
| "reward_change_std": 0.15689124166965485, |
| "reward_std": 0.4546696189790964, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.18888765573501587, |
| "step": 479 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2415.895866394043, |
| "epoch": 0.5485714285714286, |
| "grad_norm": 0.08123134076595306, |
| "kl": 1.4044344425201416e-05, |
| "lambda_div_used": 0.5890126600861549, |
| "learning_rate": 1.0482745016665526e-07, |
| "loss": 0.0739, |
| "reward": -0.36289478465914726, |
| "reward_after_mean": -0.36289478465914726, |
| "reward_after_std": 0.4815117195248604, |
| "reward_before_mean": -0.11086839716881514, |
| "reward_before_std": 0.4026770405471325, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.25202639400959015, |
| "reward_change_min": -0.37619682028889656, |
| "reward_change_std": 0.13623447716236115, |
| "reward_std": 0.4815117232501507, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.17336839414201677, |
| "step": 480 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3057.625030517578, |
| "epoch": 0.5497142857142857, |
| "grad_norm": 0.05632660165429115, |
| "kl": 2.8233975172042847e-05, |
| "lambda_div_used": 0.5841230228543282, |
| "learning_rate": 1.0437936906629334e-07, |
| "loss": 0.0198, |
| "reward": -0.386552257463336, |
| "reward_after_mean": -0.386552257463336, |
| "reward_after_std": 0.4468122199177742, |
| "reward_before_mean": -0.14256569184362888, |
| "reward_before_std": 0.38717135414481163, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.24398657120764256, |
| "reward_change_min": -0.42186837270855904, |
| "reward_change_std": 0.14736179821193218, |
| "reward_std": 0.4468122236430645, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.20506569370627403, |
| "step": 481 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2824.5833587646484, |
| "epoch": 0.5508571428571428, |
| "grad_norm": 0.0865793228149414, |
| "kl": 2.4417764507234097e-05, |
| "lambda_div_used": 0.622517004609108, |
| "learning_rate": 1.0395300688680625e-07, |
| "loss": -0.0313, |
| "reward": -0.05876433290541172, |
| "reward_after_mean": -0.05876433290541172, |
| "reward_after_std": 0.5989817604422569, |
| "reward_before_mean": 0.259432727470994, |
| "reward_before_std": 0.5737235806882381, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.31819707341492176, |
| "reward_change_min": -0.5269695781171322, |
| "reward_change_std": 0.21138717606663704, |
| "reward_std": 0.5989817790687084, |
| "rewards/accuracy_reward": 0.22916667349636555, |
| "rewards/cosine_scaled_reward": 0.030266055837273598, |
| "step": 482 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2869.250015258789, |
| "epoch": 0.552, |
| "grad_norm": 0.08511485904455185, |
| "kl": 3.474205732345581e-05, |
| "lambda_div_used": 0.6013401597738266, |
| "learning_rate": 1.0354838440848501e-07, |
| "loss": -0.0837, |
| "reward": -0.2791392467916012, |
| "reward_after_mean": -0.2791392467916012, |
| "reward_after_std": 0.5272991992533207, |
| "reward_before_mean": -0.02371177263557911, |
| "reward_before_std": 0.4689871799200773, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2554274704307318, |
| "reward_change_min": -0.40494704991579056, |
| "reward_change_std": 0.15485912468284369, |
| "reward_std": 0.5272992141544819, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.1278784405440092, |
| "step": 483 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2591.7500228881836, |
| "epoch": 0.5531428571428572, |
| "grad_norm": 0.08745326101779938, |
| "kl": 3.6368146538734436e-07, |
| "lambda_div_used": 0.6466837078332901, |
| "learning_rate": 1.0316552135205837e-07, |
| "loss": 0.0082, |
| "reward": -0.04187892563641071, |
| "reward_after_mean": -0.04187892563641071, |
| "reward_after_std": 0.6843612100929022, |
| "reward_before_mean": 0.242530676885508, |
| "reward_before_std": 0.6844684220850468, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.284409599378705, |
| "reward_change_min": -0.49204646795988083, |
| "reward_change_std": 0.1993796620517969, |
| "reward_std": 0.6843612212687731, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": -0.0283026653341949, |
| "step": 484 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2046.2500381469727, |
| "epoch": 0.5542857142857143, |
| "grad_norm": 0.09076947718858719, |
| "kl": 2.752244472503662e-05, |
| "lambda_div_used": 0.5719796344637871, |
| "learning_rate": 1.0280443637773163e-07, |
| "loss": 0.0035, |
| "reward": -0.40644849208183587, |
| "reward_after_mean": -0.40644849208183587, |
| "reward_after_std": 0.398155614733696, |
| "reward_before_mean": -0.14403727410535794, |
| "reward_before_std": 0.3282261691056192, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2624112106859684, |
| "reward_change_min": -0.43003450334072113, |
| "reward_change_std": 0.15388297475874424, |
| "reward_std": 0.39815562404692173, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.20653727487660944, |
| "step": 485 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1967.7916946411133, |
| "epoch": 0.5554285714285714, |
| "grad_norm": 0.09958811849355698, |
| "kl": 3.91155481338501e-05, |
| "lambda_div_used": 0.5703976079821587, |
| "learning_rate": 1.0246514708427701e-07, |
| "loss": 0.0966, |
| "reward": -0.26609380822628736, |
| "reward_after_mean": -0.26609380822628736, |
| "reward_after_std": 0.4386676363646984, |
| "reward_before_mean": 0.07946969009935856, |
| "reward_before_std": 0.31633214373141527, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.34556350111961365, |
| "reward_change_min": -0.48576217144727707, |
| "reward_change_std": 0.1828850321471691, |
| "reward_std": 0.4386676475405693, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.08719698037020862, |
| "step": 486 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1967.0625228881836, |
| "epoch": 0.5565714285714286, |
| "grad_norm": 0.10718576610088348, |
| "kl": 3.7085264921188354e-06, |
| "lambda_div_used": 0.6087248548865318, |
| "learning_rate": 1.0214767000817596e-07, |
| "loss": 0.0461, |
| "reward": 0.08780635055154562, |
| "reward_after_mean": 0.08780635055154562, |
| "reward_after_std": 0.6550032701343298, |
| "reward_before_mean": 0.5625860100844875, |
| "reward_before_std": 0.5011934200301766, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.47477963753044605, |
| "reward_change_min": -0.7034505233168602, |
| "reward_change_std": 0.26853484753519297, |
| "reward_std": 0.6550032943487167, |
| "rewards/accuracy_reward": 0.3958333358168602, |
| "rewards/cosine_scaled_reward": 0.16675265738740563, |
| "step": 487 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2356.8125228881836, |
| "epoch": 0.5577142857142857, |
| "grad_norm": 0.08583448082208633, |
| "kl": 2.1670013666152954e-05, |
| "lambda_div_used": 0.5598724335432053, |
| "learning_rate": 1.0185202062281336e-07, |
| "loss": -0.0446, |
| "reward": -0.39712974801659584, |
| "reward_after_mean": -0.39712974801659584, |
| "reward_after_std": 0.3282298669219017, |
| "reward_before_mean": -0.11880321707576513, |
| "reward_before_std": 0.2731999019160867, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27832652255892754, |
| "reward_change_min": -0.42709672823548317, |
| "reward_change_std": 0.16301770228892565, |
| "reward_std": 0.3282298743724823, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.20213654916733503, |
| "step": 488 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3039.2916870117188, |
| "epoch": 0.5588571428571428, |
| "grad_norm": 0.08493036031723022, |
| "kl": 4.955753684043884e-05, |
| "lambda_div_used": 0.5549413114786148, |
| "learning_rate": 1.0157821333772304e-07, |
| "loss": -0.0127, |
| "reward": -0.4395306259393692, |
| "reward_after_mean": -0.4395306259393692, |
| "reward_after_std": 0.30519232526421547, |
| "reward_before_mean": -0.16155868768692017, |
| "reward_before_std": 0.24803727120161057, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.27797193080186844, |
| "reward_change_min": -0.44194458797574043, |
| "reward_change_std": 0.1623495165258646, |
| "reward_std": 0.3051923308521509, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.22405867651104927, |
| "step": 489 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2838.0833892822266, |
| "epoch": 0.56, |
| "grad_norm": 0.06069381162524223, |
| "kl": 1.2032687664031982e-05, |
| "lambda_div_used": 0.583185076713562, |
| "learning_rate": 1.013262614978859e-07, |
| "loss": 0.0139, |
| "reward": -0.1338807214051485, |
| "reward_after_mean": -0.1338807214051485, |
| "reward_after_std": 0.4649778436869383, |
| "reward_before_mean": 0.25663699954748154, |
| "reward_before_std": 0.3789667785167694, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.39051773957908154, |
| "reward_change_min": -0.6341840960085392, |
| "reward_change_std": 0.2300328817218542, |
| "reward_std": 0.46497784554958344, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/cosine_scaled_reward": -0.014196328818798065, |
| "step": 490 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2593.7291870117188, |
| "epoch": 0.5611428571428572, |
| "grad_norm": 0.07340901345014572, |
| "kl": 3.531062975525856e-05, |
| "lambda_div_used": 0.5965912714600563, |
| "learning_rate": 1.0109617738307911e-07, |
| "loss": 0.0268, |
| "reward": 0.01746355928480625, |
| "reward_after_mean": 0.01746355928480625, |
| "reward_after_std": 0.5940692499279976, |
| "reward_before_mean": 0.47233812790364027, |
| "reward_before_std": 0.4421461224555969, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4548745583742857, |
| "reward_change_min": -0.6400704979896545, |
| "reward_change_std": 0.24487750884145498, |
| "reward_std": 0.5940692741423845, |
| "rewards/accuracy_reward": 0.3541666679084301, |
| "rewards/cosine_scaled_reward": 0.11817142926156521, |
| "step": 491 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2583.5416679382324, |
| "epoch": 0.5622857142857143, |
| "grad_norm": 0.08282249420881271, |
| "kl": 2.4866312742233276e-05, |
| "lambda_div_used": 0.5367531701922417, |
| "learning_rate": 1.0088797220727779e-07, |
| "loss": 0.0376, |
| "reward": -0.3908469006419182, |
| "reward_after_mean": -0.3908469006419182, |
| "reward_after_std": 0.27869052439928055, |
| "reward_before_mean": -0.03699151985347271, |
| "reward_before_std": 0.16138391755521297, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3538553789258003, |
| "reward_change_min": -0.5079905577003956, |
| "reward_change_std": 0.18725439626723528, |
| "reward_std": 0.27869053184986115, |
| "rewards/accuracy_reward": 0.125, |
| "rewards/cosine_scaled_reward": -0.1619915273040533, |
| "step": 492 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2176.8333740234375, |
| "epoch": 0.5634285714285714, |
| "grad_norm": 0.09936831891536713, |
| "kl": 6.021931767463684e-06, |
| "lambda_div_used": 0.596942737698555, |
| "learning_rate": 1.0070165611810855e-07, |
| "loss": 0.0246, |
| "reward": -0.037120603024959564, |
| "reward_after_mean": -0.037120603024959564, |
| "reward_after_std": 0.46812310442328453, |
| "reward_before_mean": 0.34137603268027306, |
| "reward_before_std": 0.44823691714555025, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.3784966245293617, |
| "reward_change_min": -0.5661123469471931, |
| "reward_change_std": 0.234741548076272, |
| "reward_std": 0.4681231305003166, |
| "rewards/accuracy_reward": 0.2708333432674408, |
| "rewards/cosine_scaled_reward": 0.07054269965738058, |
| "step": 493 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2352.583427429199, |
| "epoch": 0.5645714285714286, |
| "grad_norm": 0.07574566453695297, |
| "kl": 1.6648322343826294e-05, |
| "lambda_div_used": 0.6796721071004868, |
| "learning_rate": 1.005372381963547e-07, |
| "loss": 0.092, |
| "reward": 0.10963849350810051, |
| "reward_after_mean": 0.10963849350810051, |
| "reward_after_std": 0.8239130303263664, |
| "reward_before_mean": 0.4074633736163378, |
| "reward_before_std": 0.848267612978816, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2978248745203018, |
| "reward_change_min": -0.6511921100318432, |
| "reward_change_std": 0.2445714958012104, |
| "reward_std": 0.8239130582660437, |
| "rewards/accuracy_reward": 0.3125000037252903, |
| "rewards/cosine_scaled_reward": 0.09496336756274104, |
| "step": 494 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 3203.8959045410156, |
| "epoch": 0.5657142857142857, |
| "grad_norm": 0.057527996599674225, |
| "kl": 2.3846514523029327e-05, |
| "lambda_div_used": 0.5994853675365448, |
| "learning_rate": 1.0039472645551372e-07, |
| "loss": 0.0637, |
| "reward": -0.21300244145095348, |
| "reward_after_mean": -0.21300244145095348, |
| "reward_after_std": 0.48377062380313873, |
| "reward_before_mean": 0.08397414721548557, |
| "reward_before_std": 0.4531490486115217, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.29697658494114876, |
| "reward_change_min": -0.49157723411917686, |
| "reward_change_std": 0.1875515878200531, |
| "reward_std": 0.4837706368416548, |
| "rewards/accuracy_reward": 0.16666667349636555, |
| "rewards/cosine_scaled_reward": -0.08269252139143646, |
| "step": 495 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 1895.9375457763672, |
| "epoch": 0.5668571428571428, |
| "grad_norm": 0.10791665315628052, |
| "kl": 1.890142448246479e-05, |
| "lambda_div_used": 0.6198414713144302, |
| "learning_rate": 1.002741278414069e-07, |
| "loss": 0.0183, |
| "reward": 0.09977892541792244, |
| "reward_after_mean": 0.09977892541792244, |
| "reward_after_std": 0.6343465056270361, |
| "reward_before_mean": 0.5255357641726732, |
| "reward_before_std": 0.555769513361156, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.4257568307220936, |
| "reward_change_min": -0.6588771641254425, |
| "reward_change_std": 0.25742017664015293, |
| "reward_std": 0.634346516802907, |
| "rewards/accuracy_reward": 0.35416667349636555, |
| "rewards/cosine_scaled_reward": 0.17136909160763025, |
| "step": 496 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2577.645851135254, |
| "epoch": 0.568, |
| "grad_norm": 0.06887742131948471, |
| "kl": 2.493336796760559e-05, |
| "lambda_div_used": 0.6232517957687378, |
| "learning_rate": 1.0017544823184055e-07, |
| "loss": 0.0143, |
| "reward": 0.039356768131256104, |
| "reward_after_mean": 0.039356768131256104, |
| "reward_after_std": 0.5658366903662682, |
| "reward_before_mean": 0.401943476870656, |
| "reward_before_std": 0.5692849718034267, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.362586697563529, |
| "reward_change_min": -0.6212925836443901, |
| "reward_change_std": 0.24119753576815128, |
| "reward_std": 0.5658366959542036, |
| "rewards/accuracy_reward": 0.3333333469927311, |
| "rewards/cosine_scaled_reward": 0.06861014291644096, |
| "step": 497 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2857.2916946411133, |
| "epoch": 0.5691428571428572, |
| "grad_norm": 0.06479962170124054, |
| "kl": 9.963754564523697e-06, |
| "lambda_div_used": 0.6236685812473297, |
| "learning_rate": 1.0009869243631952e-07, |
| "loss": 0.0008, |
| "reward": -0.18094617873430252, |
| "reward_after_mean": -0.18094617873430252, |
| "reward_after_std": 0.6155845355242491, |
| "reward_before_mean": 0.07468715589493513, |
| "reward_before_std": 0.5768492119386792, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2556333262473345, |
| "reward_change_min": -0.4431779384613037, |
| "reward_change_std": 0.16846577636897564, |
| "reward_std": 0.6155845616012812, |
| "rewards/accuracy_reward": 0.1666666679084301, |
| "rewards/cosine_scaled_reward": -0.0919795110821724, |
| "step": 498 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2707.791702270508, |
| "epoch": 0.5702857142857143, |
| "grad_norm": 0.06317302584648132, |
| "kl": 1.5532365068793297e-05, |
| "lambda_div_used": 0.6250721588730812, |
| "learning_rate": 1.000438641958131e-07, |
| "loss": -0.0298, |
| "reward": -0.12677906453609467, |
| "reward_after_mean": -0.12677906453609467, |
| "reward_after_std": 0.6044176463037729, |
| "reward_before_mean": 0.1698420336470008, |
| "reward_before_std": 0.5829241154715419, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.2966211009770632, |
| "reward_change_min": -0.5905468240380287, |
| "reward_change_std": 0.2108100038021803, |
| "reward_std": 0.6044176481664181, |
| "rewards/accuracy_reward": 0.2083333358168602, |
| "rewards/cosine_scaled_reward": -0.03849130589514971, |
| "step": 499 |
| }, |
| { |
| "clip_fraction": 0.0, |
| "completion_length": 2980.9167098999023, |
| "epoch": 0.5714285714285714, |
| "grad_norm": 0.0621151477098465, |
| "kl": 1.033581793308258e-05, |
| "lambda_div_used": 0.6023986041545868, |
| "learning_rate": 1.0001096618257236e-07, |
| "loss": 0.0314, |
| "reward": -0.26747864981007297, |
| "reward_after_mean": -0.26747864981007297, |
| "reward_after_std": 0.5251844674348831, |
| "reward_before_mean": -0.0025623496621847153, |
| "reward_before_std": 0.4696931503713131, |
| "reward_change_max": 0.0, |
| "reward_change_mean": -0.26491627655923367, |
| "reward_change_min": -0.43205036222934723, |
| "reward_change_std": 0.15884541906416416, |
| "reward_std": 0.5251844730228186, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.10672901570796967, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.5714285714285714, |
| "step": 500, |
| "total_flos": 0.0, |
| "train_loss": 0.005314271912367986, |
| "train_runtime": 106074.3369, |
| "train_samples_per_second": 0.226, |
| "train_steps_per_second": 0.005 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 500, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 10, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 6, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
