| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.856898029134533, |
| "eval_steps": 500, |
| "global_step": 500, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1549.7083435058594, |
| "epoch": 0.001713796058269066, |
| "grad_norm": 0.2566092312335968, |
| "kl": 0.0, |
| "learning_rate": 2e-08, |
| "loss": -0.0569, |
| "reward": 0.6208184361457825, |
| "reward_std": 0.9855608642101288, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.18897756189107895, |
| "rewards/format_reward": 0.2916666753590107, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1414.0417175292969, |
| "epoch": 0.003427592116538132, |
| "grad_norm": 0.29864758253097534, |
| "kl": 0.0, |
| "learning_rate": 4e-08, |
| "loss": -0.0809, |
| "reward": 0.1427215114235878, |
| "reward_std": 0.6335150748491287, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.25154703110456467, |
| "rewards/format_reward": 0.2291666753590107, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1390.2917175292969, |
| "epoch": 0.005141388174807198, |
| "grad_norm": 0.29091691970825195, |
| "kl": 9.447336196899414e-05, |
| "learning_rate": 6e-08, |
| "loss": 0.0646, |
| "reward": 0.4808660186827183, |
| "reward_std": 0.47989463061094284, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.20031768828630447, |
| "rewards/format_reward": 0.2812500102445483, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1567.9375305175781, |
| "epoch": 0.006855184233076264, |
| "grad_norm": 0.26566898822784424, |
| "kl": 7.075071334838867e-05, |
| "learning_rate": 8e-08, |
| "loss": -0.0258, |
| "reward": 0.18869880307465792, |
| "reward_std": 0.7589461207389832, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.21264867670834064, |
| "rewards/format_reward": 0.1979166716337204, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1479.9167175292969, |
| "epoch": 0.00856898029134533, |
| "grad_norm": 0.2975413203239441, |
| "kl": 7.289648056030273e-05, |
| "learning_rate": 1e-07, |
| "loss": -0.0589, |
| "reward": 0.8189541846513748, |
| "reward_std": 0.8007446154952049, |
| "rewards/accuracy_reward": 0.09375000186264515, |
| "rewards/cosine_scaled_reward": -0.1580449752509594, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1646.7917175292969, |
| "epoch": 0.010282776349614395, |
| "grad_norm": 0.23834176361560822, |
| "kl": 8.058547973632812e-05, |
| "learning_rate": 1.2e-07, |
| "loss": 0.0029, |
| "reward": -0.04225504118949175, |
| "reward_std": 0.553499698638916, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2690293677151203, |
| "rewards/format_reward": 0.17708334140479565, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1165.6458435058594, |
| "epoch": 0.011996572407883462, |
| "grad_norm": 0.3262133002281189, |
| "kl": 0.00012004375457763672, |
| "learning_rate": 1.4e-07, |
| "loss": -0.0617, |
| "reward": 1.1578989699482918, |
| "reward_std": 0.9958820343017578, |
| "rewards/accuracy_reward": 0.09375000279396772, |
| "rewards/cosine_scaled_reward": -0.11706694308668375, |
| "rewards/format_reward": 0.4583333507180214, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1534.3958740234375, |
| "epoch": 0.013710368466152529, |
| "grad_norm": 0.24637284874916077, |
| "kl": 6.645917892456055e-05, |
| "learning_rate": 1.6e-07, |
| "loss": -0.0325, |
| "reward": 0.790864821523428, |
| "reward_std": 1.2760685980319977, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.1795810554176569, |
| "rewards/format_reward": 0.3229166753590107, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1406.0833740234375, |
| "epoch": 0.015424164524421594, |
| "grad_norm": 0.3252864181995392, |
| "kl": 6.35385513305664e-05, |
| "learning_rate": 1.8e-07, |
| "loss": 0.0611, |
| "reward": 0.46893399208784103, |
| "reward_std": 0.7476691864430904, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.21526261791586876, |
| "rewards/format_reward": 0.2812500111758709, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1481.1042022705078, |
| "epoch": 0.01713796058269066, |
| "grad_norm": 0.3649034798145294, |
| "kl": 0.00012791156768798828, |
| "learning_rate": 2e-07, |
| "loss": 0.0039, |
| "reward": 0.5377852376550436, |
| "reward_std": 0.8327888622879982, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.17093832325190306, |
| "rewards/format_reward": 0.2187500111758709, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1507.2500305175781, |
| "epoch": 0.018851756640959727, |
| "grad_norm": 0.3549154996871948, |
| "kl": 8.279085159301758e-05, |
| "learning_rate": 2.1999999999999998e-07, |
| "loss": 0.0096, |
| "reward": 0.625231646001339, |
| "reward_std": 0.7666523866355419, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.17873376421630383, |
| "rewards/format_reward": 0.2291666753590107, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1587.4167175292969, |
| "epoch": 0.02056555269922879, |
| "grad_norm": 0.251277357339859, |
| "kl": 7.414817810058594e-05, |
| "learning_rate": 2.4e-07, |
| "loss": -0.031, |
| "reward": 0.2848171964287758, |
| "reward_std": 0.8330317437648773, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.2284027300775051, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1385.5000305175781, |
| "epoch": 0.022279348757497857, |
| "grad_norm": 0.3666461408138275, |
| "kl": 8.702278137207031e-05, |
| "learning_rate": 2.6e-07, |
| "loss": 0.104, |
| "reward": 0.657318189740181, |
| "reward_std": 0.7007318809628487, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.23300136625766754, |
| "rewards/format_reward": 0.3854166744276881, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1418.2917175292969, |
| "epoch": 0.023993144815766924, |
| "grad_norm": 0.30520063638687134, |
| "kl": 0.00011420249938964844, |
| "learning_rate": 2.8e-07, |
| "loss": -0.0065, |
| "reward": 0.4347895681858063, |
| "reward_std": 0.5189282819628716, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.1626796661876142, |
| "rewards/format_reward": 0.1666666716337204, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1485.1875305175781, |
| "epoch": 0.02570694087403599, |
| "grad_norm": 0.2740783393383026, |
| "kl": 0.00011640787124633789, |
| "learning_rate": 3e-07, |
| "loss": -0.0048, |
| "reward": -0.019001017324626446, |
| "reward_std": 0.6500238478183746, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.26481105387210846, |
| "rewards/format_reward": 0.13541667442768812, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1309.9167175292969, |
| "epoch": 0.027420736932305057, |
| "grad_norm": 0.3026612102985382, |
| "kl": 7.218122482299805e-05, |
| "learning_rate": 3.2e-07, |
| "loss": 0.023, |
| "reward": 0.9514074325561523, |
| "reward_std": 0.8899444937705994, |
| "rewards/accuracy_reward": 0.09375, |
| "rewards/cosine_scaled_reward": -0.17226803209632635, |
| "rewards/format_reward": 0.4270833358168602, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1449.9166870117188, |
| "epoch": 0.02913453299057412, |
| "grad_norm": 0.27961981296539307, |
| "kl": 7.82012939453125e-05, |
| "learning_rate": 3.4000000000000003e-07, |
| "loss": 0.0057, |
| "reward": 0.5791401639580727, |
| "reward_std": 0.6529509723186493, |
| "rewards/accuracy_reward": 0.08333333488553762, |
| "rewards/cosine_scaled_reward": -0.19404403865337372, |
| "rewards/format_reward": 0.18750000651925802, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1511.6250305175781, |
| "epoch": 0.030848329048843187, |
| "grad_norm": 0.2333238124847412, |
| "kl": 8.535385131835938e-05, |
| "learning_rate": 3.6e-07, |
| "loss": 0.0385, |
| "reward": 0.3788021163782105, |
| "reward_std": 0.9533726572990417, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.20547283068299294, |
| "rewards/format_reward": 0.1770833395421505, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1607.5833740234375, |
| "epoch": 0.032562125107112254, |
| "grad_norm": 0.3147532343864441, |
| "kl": 8.320808410644531e-05, |
| "learning_rate": 3.7999999999999996e-07, |
| "loss": 0.0074, |
| "reward": 0.3463876098394394, |
| "reward_std": 0.7631522864103317, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2376747578382492, |
| "rewards/format_reward": 0.3020833432674408, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1680.8333740234375, |
| "epoch": 0.03427592116538132, |
| "grad_norm": 0.3314201533794403, |
| "kl": 6.365776062011719e-05, |
| "learning_rate": 4e-07, |
| "loss": 0.0129, |
| "reward": 0.08592129033058882, |
| "reward_std": 0.8577414005994797, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.26088032126426697, |
| "rewards/format_reward": 0.1354166716337204, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1422.8541870117188, |
| "epoch": 0.03598971722365039, |
| "grad_norm": 0.2986367642879486, |
| "kl": 8.559226989746094e-05, |
| "learning_rate": 4.1999999999999995e-07, |
| "loss": -0.1152, |
| "reward": 0.545179876498878, |
| "reward_std": 0.853977307677269, |
| "rewards/accuracy_reward": 0.07291666977107525, |
| "rewards/cosine_scaled_reward": -0.23388315364718437, |
| "rewards/format_reward": 0.2812500176951289, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1561.0833740234375, |
| "epoch": 0.037703513281919454, |
| "grad_norm": 0.33064112067222595, |
| "kl": 9.429454803466797e-05, |
| "learning_rate": 4.3999999999999997e-07, |
| "loss": 0.0381, |
| "reward": -0.020190313458442688, |
| "reward_std": 0.42865169048309326, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2501176781952381, |
| "rewards/format_reward": 0.15625000558793545, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1617.5000305175781, |
| "epoch": 0.03941730934018852, |
| "grad_norm": 0.24645616114139557, |
| "kl": 5.990266799926758e-05, |
| "learning_rate": 4.6e-07, |
| "loss": 0.0022, |
| "reward": -0.061072273179888725, |
| "reward_std": 0.5299023687839508, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.32925406843423843, |
| "rewards/format_reward": 0.2500000111758709, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1456.9375305175781, |
| "epoch": 0.04113110539845758, |
| "grad_norm": 0.2407306581735611, |
| "kl": 7.05718994140625e-05, |
| "learning_rate": 4.8e-07, |
| "loss": -0.0391, |
| "reward": 1.0003801509737968, |
| "reward_std": 0.8640259802341461, |
| "rewards/accuracy_reward": 0.11458333488553762, |
| "rewards/cosine_scaled_reward": -0.15404087863862514, |
| "rewards/format_reward": 0.32291666977107525, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1410.9792175292969, |
| "epoch": 0.04284490145672665, |
| "grad_norm": 0.2708369791507721, |
| "kl": 8.213520050048828e-05, |
| "learning_rate": 5e-07, |
| "loss": 0.0349, |
| "reward": 0.4050113819539547, |
| "reward_std": 0.7914649695158005, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.19529718905687332, |
| "rewards/format_reward": 0.31250000558793545, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1593.2917175292969, |
| "epoch": 0.044558697514995714, |
| "grad_norm": 0.27268657088279724, |
| "kl": 6.699562072753906e-05, |
| "learning_rate": 5.2e-07, |
| "loss": 0.0223, |
| "reward": 0.3962297812104225, |
| "reward_std": 0.945412665605545, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.24394675344228745, |
| "rewards/format_reward": 0.2604166753590107, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1592.9375305175781, |
| "epoch": 0.04627249357326478, |
| "grad_norm": 0.25189143419265747, |
| "kl": 7.450580596923828e-05, |
| "learning_rate": 5.4e-07, |
| "loss": -0.035, |
| "reward": 0.3084355629980564, |
| "reward_std": 0.7038140445947647, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.19312112405896187, |
| "rewards/format_reward": 0.19791667442768812, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1407.7500305175781, |
| "epoch": 0.04798628963153385, |
| "grad_norm": 0.276398241519928, |
| "kl": 9.196996688842773e-05, |
| "learning_rate": 5.6e-07, |
| "loss": -0.0933, |
| "reward": 1.1628794074058533, |
| "reward_std": 1.145683228969574, |
| "rewards/accuracy_reward": 0.10416666697710752, |
| "rewards/cosine_scaled_reward": -0.15041928738355637, |
| "rewards/format_reward": 0.4062500149011612, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1545.0000305175781, |
| "epoch": 0.049700085689802914, |
| "grad_norm": 0.2882435619831085, |
| "kl": 6.723403930664062e-05, |
| "learning_rate": 5.8e-07, |
| "loss": 0.0013, |
| "reward": 0.19917990267276764, |
| "reward_std": 0.7036862522363663, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2851256728172302, |
| "rewards/format_reward": 0.22916666977107525, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1446.9375, |
| "epoch": 0.05141388174807198, |
| "grad_norm": 0.3235947787761688, |
| "kl": 9.739398956298828e-05, |
| "learning_rate": 6e-07, |
| "loss": -0.0015, |
| "reward": 0.8554540276527405, |
| "reward_std": 0.9236274063587189, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.1764096152037382, |
| "rewards/format_reward": 0.385416679084301, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1410.3750305175781, |
| "epoch": 0.05312767780634105, |
| "grad_norm": 0.35821714997291565, |
| "kl": 6.109476089477539e-05, |
| "learning_rate": 6.2e-07, |
| "loss": 0.1537, |
| "reward": 0.8505711704492569, |
| "reward_std": 0.775887742638588, |
| "rewards/accuracy_reward": 0.10416666697710752, |
| "rewards/cosine_scaled_reward": -0.18543021008372307, |
| "rewards/format_reward": 0.27083333767950535, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1427.0000610351562, |
| "epoch": 0.054841473864610114, |
| "grad_norm": 0.29756107926368713, |
| "kl": 8.952617645263672e-05, |
| "learning_rate": 6.4e-07, |
| "loss": 0.0014, |
| "reward": 0.2789556197822094, |
| "reward_std": 0.6537229269742966, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.25284862518310547, |
| "rewards/format_reward": 0.28125000558793545, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1226.0625457763672, |
| "epoch": 0.056555269922879174, |
| "grad_norm": 0.311769038438797, |
| "kl": 5.3822994232177734e-05, |
| "learning_rate": 6.6e-07, |
| "loss": 0.0657, |
| "reward": 1.2427364438772202, |
| "reward_std": 0.8947374746203423, |
| "rewards/accuracy_reward": 0.12500000465661287, |
| "rewards/cosine_scaled_reward": -0.14077210240066051, |
| "rewards/format_reward": 0.4062500149011612, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1463.5625610351562, |
| "epoch": 0.05826906598114824, |
| "grad_norm": 0.2749904990196228, |
| "kl": 7.456541061401367e-05, |
| "learning_rate": 6.800000000000001e-07, |
| "loss": 0.0151, |
| "reward": 0.2089916095137596, |
| "reward_std": 0.7645938321948051, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2636757530272007, |
| "rewards/format_reward": 0.2083333358168602, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1574.1250610351562, |
| "epoch": 0.05998286203941731, |
| "grad_norm": 0.2814057469367981, |
| "kl": 6.365776062011719e-05, |
| "learning_rate": 7e-07, |
| "loss": -0.0354, |
| "reward": 0.4698517946526408, |
| "reward_std": 0.7770050093531609, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.2284439541399479, |
| "rewards/format_reward": 0.3125000149011612, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1329.6458740234375, |
| "epoch": 0.061696658097686374, |
| "grad_norm": 0.29900285601615906, |
| "kl": 0.00013148784637451172, |
| "learning_rate": 7.2e-07, |
| "loss": 0.0355, |
| "reward": 0.38275275379419327, |
| "reward_std": 0.5484570488333702, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.2323998138308525, |
| "rewards/format_reward": 0.291666679084301, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1505.4583740234375, |
| "epoch": 0.06341045415595545, |
| "grad_norm": 0.23520199954509735, |
| "kl": 8.672475814819336e-05, |
| "learning_rate": 7.4e-07, |
| "loss": 0.0243, |
| "reward": 0.3799854665994644, |
| "reward_std": 0.775396078824997, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.2143558170646429, |
| "rewards/format_reward": 0.16666667442768812, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1575.1041870117188, |
| "epoch": 0.06512425021422451, |
| "grad_norm": 0.28803354501724243, |
| "kl": 6.490945816040039e-05, |
| "learning_rate": 7.599999999999999e-07, |
| "loss": 0.0042, |
| "reward": 0.08564997464418411, |
| "reward_std": 0.524849109351635, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2424239031970501, |
| "rewards/format_reward": 0.20833333767950535, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1587.0625305175781, |
| "epoch": 0.06683804627249357, |
| "grad_norm": 0.2588130235671997, |
| "kl": 7.474422454833984e-05, |
| "learning_rate": 7.799999999999999e-07, |
| "loss": 0.0004, |
| "reward": 0.31661053746938705, |
| "reward_std": 0.6547420993447304, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.211146367713809, |
| "rewards/format_reward": 0.17708333861082792, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1419.9375610351562, |
| "epoch": 0.06855184233076264, |
| "grad_norm": 0.27477696537971497, |
| "kl": 6.985664367675781e-05, |
| "learning_rate": 8e-07, |
| "loss": -0.1003, |
| "reward": 1.1732516959309578, |
| "reward_std": 0.8123161420226097, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/cosine_scaled_reward": -0.08519345941022038, |
| "rewards/format_reward": 0.2708333386108279, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1474.6458740234375, |
| "epoch": 0.0702656383890317, |
| "grad_norm": 0.27839529514312744, |
| "kl": 7.69495964050293e-05, |
| "learning_rate": 8.199999999999999e-07, |
| "loss": 0.0605, |
| "reward": 1.1124837547540665, |
| "reward_std": 1.0182685256004333, |
| "rewards/accuracy_reward": 0.11458333488553762, |
| "rewards/cosine_scaled_reward": -0.15390180423855782, |
| "rewards/format_reward": 0.3125000074505806, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1352.8333740234375, |
| "epoch": 0.07197943444730077, |
| "grad_norm": 0.2626473307609558, |
| "kl": 5.447864532470703e-05, |
| "learning_rate": 8.399999999999999e-07, |
| "loss": -0.0291, |
| "reward": 0.4686532625928521, |
| "reward_std": 0.6282949075102806, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.24288901686668396, |
| "rewards/format_reward": 0.4479166753590107, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1543.5625610351562, |
| "epoch": 0.07369323050556983, |
| "grad_norm": 0.2575068473815918, |
| "kl": 6.371736526489258e-05, |
| "learning_rate": 8.599999999999999e-07, |
| "loss": 0.0766, |
| "reward": 1.0126795023679733, |
| "reward_std": 0.9010422676801682, |
| "rewards/accuracy_reward": 0.1145833358168602, |
| "rewards/cosine_scaled_reward": -0.14877209067344666, |
| "rewards/format_reward": 0.3333333386108279, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1446.5833740234375, |
| "epoch": 0.07540702656383891, |
| "grad_norm": 0.2544824481010437, |
| "kl": 5.650520324707031e-05, |
| "learning_rate": 8.799999999999999e-07, |
| "loss": -0.0808, |
| "reward": 0.5173629745841026, |
| "reward_std": 0.7816298678517342, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.21039017662405968, |
| "rewards/format_reward": 0.3750000037252903, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1529.8542175292969, |
| "epoch": 0.07712082262210797, |
| "grad_norm": 0.26859909296035767, |
| "kl": 0.00012755393981933594, |
| "learning_rate": 9e-07, |
| "loss": 0.0018, |
| "reward": 0.5219013094902039, |
| "reward_std": 0.7768488749861717, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.14432941935956478, |
| "rewards/format_reward": 0.2083333432674408, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1248.5000610351562, |
| "epoch": 0.07883461868037704, |
| "grad_norm": 0.37795937061309814, |
| "kl": 0.00010341405868530273, |
| "learning_rate": 9.2e-07, |
| "loss": 0.0897, |
| "reward": 0.8892228305339813, |
| "reward_std": 0.9948980510234833, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.18321672268211842, |
| "rewards/format_reward": 0.4791666865348816, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1435.1875305175781, |
| "epoch": 0.0805484147386461, |
| "grad_norm": 0.262204110622406, |
| "kl": 9.328126907348633e-05, |
| "learning_rate": 9.399999999999999e-07, |
| "loss": -0.0252, |
| "reward": 0.5409297198057175, |
| "reward_std": 0.761884793639183, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.2008819542825222, |
| "rewards/format_reward": 0.322916679084301, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1400.5000305175781, |
| "epoch": 0.08226221079691516, |
| "grad_norm": 0.2843024432659149, |
| "kl": 7.283687591552734e-05, |
| "learning_rate": 9.6e-07, |
| "loss": 0.0258, |
| "reward": 0.81534643471241, |
| "reward_std": 1.0338954292237759, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.15888767689466476, |
| "rewards/format_reward": 0.3750000074505806, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1199.875, |
| "epoch": 0.08397600685518423, |
| "grad_norm": 0.35097768902778625, |
| "kl": 0.00010097026824951172, |
| "learning_rate": 9.8e-07, |
| "loss": -0.0198, |
| "reward": 1.327251985669136, |
| "reward_std": 1.300454005599022, |
| "rewards/accuracy_reward": 0.11458333488553762, |
| "rewards/cosine_scaled_reward": -0.12861149292439222, |
| "rewards/format_reward": 0.510416679084301, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1303.0, |
| "epoch": 0.0856898029134533, |
| "grad_norm": 0.34222906827926636, |
| "kl": 0.00014340877532958984, |
| "learning_rate": 1e-06, |
| "loss": 0.1216, |
| "reward": 0.4692706950008869, |
| "reward_std": 0.7595674097537994, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2138059325516224, |
| "rewards/format_reward": 0.4270833507180214, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1519.9166870117188, |
| "epoch": 0.08740359897172237, |
| "grad_norm": 0.24351148307323456, |
| "kl": 6.276369094848633e-05, |
| "learning_rate": 9.999890338174275e-07, |
| "loss": -0.0212, |
| "reward": 0.6564827468246222, |
| "reward_std": 0.5184785276651382, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.18480780348181725, |
| "rewards/format_reward": 0.2500000037252903, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1529.5833740234375, |
| "epoch": 0.08911739502999143, |
| "grad_norm": 0.2733688950538635, |
| "kl": 9.131431579589844e-05, |
| "learning_rate": 9.999561358041868e-07, |
| "loss": -0.0305, |
| "reward": 0.31063079461455345, |
| "reward_std": 0.9850436672568321, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2166740708053112, |
| "rewards/format_reward": 0.17708333861082792, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1467.9166870117188, |
| "epoch": 0.0908311910882605, |
| "grad_norm": 0.25451886653900146, |
| "kl": 0.00011324882507324219, |
| "learning_rate": 9.999013075636804e-07, |
| "loss": 0.0124, |
| "reward": 0.25967272371053696, |
| "reward_std": 0.6069408655166626, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.25990210846066475, |
| "rewards/format_reward": 0.28125, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1637.4791870117188, |
| "epoch": 0.09254498714652956, |
| "grad_norm": 0.23521429300308228, |
| "kl": 4.8041343688964844e-05, |
| "learning_rate": 9.998245517681593e-07, |
| "loss": -0.0419, |
| "reward": -0.009522376582026482, |
| "reward_std": 0.6713765487074852, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.25045548751950264, |
| "rewards/format_reward": 0.15625000558793545, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1639.5625305175781, |
| "epoch": 0.09425878320479864, |
| "grad_norm": 0.2962999641895294, |
| "kl": 6.145238876342773e-05, |
| "learning_rate": 9.997258721585931e-07, |
| "loss": 0.0274, |
| "reward": 0.05718616710510105, |
| "reward_std": 0.5747075229883194, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.23509091138839722, |
| "rewards/format_reward": 0.1666666679084301, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1558.6458740234375, |
| "epoch": 0.0959725792630677, |
| "grad_norm": 0.3224487602710724, |
| "kl": 0.0001246929168701172, |
| "learning_rate": 9.996052735444862e-07, |
| "loss": -0.0361, |
| "reward": 0.555650869384408, |
| "reward_std": 0.9909381493926048, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.18972268514335155, |
| "rewards/format_reward": 0.27083334140479565, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1691.5833740234375, |
| "epoch": 0.09768637532133675, |
| "grad_norm": 0.25908270478248596, |
| "kl": 5.307793617248535e-05, |
| "learning_rate": 9.994627618036452e-07, |
| "loss": 0.0389, |
| "reward": 0.4178902134299278, |
| "reward_std": 0.6143478825688362, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.16642580553889275, |
| "rewards/format_reward": 0.1875000074505806, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1550.7083435058594, |
| "epoch": 0.09940017137960583, |
| "grad_norm": 0.26480039954185486, |
| "kl": 9.274482727050781e-05, |
| "learning_rate": 9.992983438818915e-07, |
| "loss": 0.0277, |
| "reward": 0.3206222988665104, |
| "reward_std": 0.7064706832170486, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.19392597675323486, |
| "rewards/format_reward": 0.23958333395421505, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1367.5208435058594, |
| "epoch": 0.10111396743787489, |
| "grad_norm": 0.3325769007205963, |
| "kl": 0.00011909008026123047, |
| "learning_rate": 9.991120277927223e-07, |
| "loss": -0.0285, |
| "reward": 0.725213622674346, |
| "reward_std": 0.788853645324707, |
| "rewards/accuracy_reward": 0.08333333488553762, |
| "rewards/cosine_scaled_reward": -0.21251679956912994, |
| "rewards/format_reward": 0.2812500149011612, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1617.6041870117188, |
| "epoch": 0.10282776349614396, |
| "grad_norm": 0.26701754331588745, |
| "kl": 7.194280624389648e-05, |
| "learning_rate": 9.989038226169207e-07, |
| "loss": 0.0283, |
| "reward": 0.17363526113331318, |
| "reward_std": 0.38850264623761177, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.1875241920351982, |
| "rewards/format_reward": 0.13541666977107525, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1648.0416870117188, |
| "epoch": 0.10454155955441302, |
| "grad_norm": 0.33412033319473267, |
| "kl": 6.657838821411133e-05, |
| "learning_rate": 9.98673738502114e-07, |
| "loss": -0.0192, |
| "reward": 0.20339430589228868, |
| "reward_std": 0.7374705746769905, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.23108146712183952, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1308.2500305175781, |
| "epoch": 0.1062553556126821, |
| "grad_norm": 0.3093053102493286, |
| "kl": 0.0001016855239868164, |
| "learning_rate": 9.98421786662277e-07, |
| "loss": -0.1007, |
| "reward": 1.0369166433811188, |
| "reward_std": 1.086327701807022, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.15324707701802254, |
| "rewards/format_reward": 0.354166679084301, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1690.1458740234375, |
| "epoch": 0.10796915167095116, |
| "grad_norm": 0.2637367248535156, |
| "kl": 7.319450378417969e-05, |
| "learning_rate": 9.981479793771866e-07, |
| "loss": 0.0201, |
| "reward": -0.04099972918629646, |
| "reward_std": 0.5225711427628994, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2726781591773033, |
| "rewards/format_reward": 0.1354166716337204, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1428.8333740234375, |
| "epoch": 0.10968294772922023, |
| "grad_norm": 0.31131911277770996, |
| "kl": 0.00010216236114501953, |
| "learning_rate": 9.97852329991824e-07, |
| "loss": 0.0963, |
| "reward": 0.3072669580578804, |
| "reward_std": 0.6717003732919693, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.24537866562604904, |
| "rewards/format_reward": 0.250000006519258, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1547.7917175292969, |
| "epoch": 0.11139674378748929, |
| "grad_norm": 0.3028079569339752, |
| "kl": 9.393692016601562e-05, |
| "learning_rate": 9.975348529157229e-07, |
| "loss": -0.0456, |
| "reward": 0.39046264067292213, |
| "reward_std": 0.4245295375585556, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.18120174296200275, |
| "rewards/format_reward": 0.09375000279396772, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1437.0417175292969, |
| "epoch": 0.11311053984575835, |
| "grad_norm": 0.34703078866004944, |
| "kl": 7.510185241699219e-05, |
| "learning_rate": 9.971955636222684e-07, |
| "loss": -0.0542, |
| "reward": 1.3410741090774536, |
| "reward_std": 0.6576060205698013, |
| "rewards/accuracy_reward": 0.13541667070239782, |
| "rewards/cosine_scaled_reward": -0.11065312474966049, |
| "rewards/format_reward": 0.354166679084301, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1505.8542175292969, |
| "epoch": 0.11482433590402742, |
| "grad_norm": 0.2783399224281311, |
| "kl": 7.730722427368164e-05, |
| "learning_rate": 9.968344786479415e-07, |
| "loss": 0.0006, |
| "reward": 0.2658543325960636, |
| "reward_std": 0.807809866964817, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2537504807114601, |
| "rewards/format_reward": 0.250000006519258, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1534.6666870117188, |
| "epoch": 0.11653813196229648, |
| "grad_norm": 0.2961093783378601, |
| "kl": 9.751319885253906e-05, |
| "learning_rate": 9.964516155915151e-07, |
| "loss": 0.0957, |
| "reward": 0.23031030222773552, |
| "reward_std": 0.6089238375425339, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.20732926949858665, |
| "rewards/format_reward": 0.22916667349636555, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1538.8750610351562, |
| "epoch": 0.11825192802056556, |
| "grad_norm": 0.2661342918872833, |
| "kl": 9.822845458984375e-05, |
| "learning_rate": 9.960469931131936e-07, |
| "loss": -0.0247, |
| "reward": 0.4905584305524826, |
| "reward_std": 1.0583526492118835, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.20718198269605637, |
| "rewards/format_reward": 0.2708333395421505, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1440.1875610351562, |
| "epoch": 0.11996572407883462, |
| "grad_norm": 0.28563302755355835, |
| "kl": 9.071826934814453e-05, |
| "learning_rate": 9.956206309337066e-07, |
| "loss": -0.0214, |
| "reward": 1.1916613206267357, |
| "reward_std": 0.6305195689201355, |
| "rewards/accuracy_reward": 0.12500000651925802, |
| "rewards/cosine_scaled_reward": -0.1197041841223836, |
| "rewards/format_reward": 0.354166679084301, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1498.0000610351562, |
| "epoch": 0.12167952013710369, |
| "grad_norm": 0.3835605978965759, |
| "kl": 7.128715515136719e-05, |
| "learning_rate": 9.951725498333448e-07, |
| "loss": 0.0491, |
| "reward": 0.7751804813742638, |
| "reward_std": 1.0542180240154266, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.17253165692090988, |
| "rewards/format_reward": 0.2291666716337204, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1348.8125610351562, |
| "epoch": 0.12339331619537275, |
| "grad_norm": 0.3369770646095276, |
| "kl": 0.00010955333709716797, |
| "learning_rate": 9.947027716509488e-07, |
| "loss": -0.0223, |
| "reward": 1.3596730416174978, |
| "reward_std": 1.071122221648693, |
| "rewards/accuracy_reward": 0.1354166679084301, |
| "rewards/cosine_scaled_reward": -0.1284898892045021, |
| "rewards/format_reward": 0.4270833469927311, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1523.4167175292969, |
| "epoch": 0.12510711225364182, |
| "grad_norm": 0.38462695479393005, |
| "kl": 0.00013840198516845703, |
| "learning_rate": 9.942113192828444e-07, |
| "loss": 0.0507, |
| "reward": 0.7955479570664465, |
| "reward_std": 0.7627622112631798, |
| "rewards/accuracy_reward": 0.09375, |
| "rewards/cosine_scaled_reward": -0.18039103038609028, |
| "rewards/format_reward": 0.2708333460614085, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1232.5625457763672, |
| "epoch": 0.1268209083119109, |
| "grad_norm": 0.3747916519641876, |
| "kl": 0.00013077259063720703, |
| "learning_rate": 9.93698216681727e-07, |
| "loss": -0.0132, |
| "reward": 1.1943221688270569, |
| "reward_std": 1.380107969045639, |
| "rewards/accuracy_reward": 0.11458333395421505, |
| "rewards/cosine_scaled_reward": -0.1479830238968134, |
| "rewards/format_reward": 0.46875, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1585.6250305175781, |
| "epoch": 0.12853470437017994, |
| "grad_norm": 0.2654974162578583, |
| "kl": 0.0001055002212524414, |
| "learning_rate": 9.931634888554935e-07, |
| "loss": 0.0357, |
| "reward": 0.0786585733294487, |
| "reward_std": 0.7322740331292152, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2772916667163372, |
| "rewards/format_reward": 0.15625000558793545, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1137.7708435058594, |
| "epoch": 0.13024850042844902, |
| "grad_norm": 0.33660879731178284, |
| "kl": 0.0001277923583984375, |
| "learning_rate": 9.926071618660237e-07, |
| "loss": 0.0421, |
| "reward": 1.1408534049987793, |
| "reward_std": 0.6549174636602402, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.1487407712265849, |
| "rewards/format_reward": 0.5000000111758709, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1504.2708740234375, |
| "epoch": 0.1319622964867181, |
| "grad_norm": 0.292646586894989, |
| "kl": 0.00012028217315673828, |
| "learning_rate": 9.9202926282791e-07, |
| "loss": 0.1113, |
| "reward": 0.42379511147737503, |
| "reward_std": 0.9228705763816833, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.18780511617660522, |
| "rewards/format_reward": 0.23958334140479565, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1620.0416870117188, |
| "epoch": 0.13367609254498714, |
| "grad_norm": 0.2709222435951233, |
| "kl": 0.00012493133544921875, |
| "learning_rate": 9.91429819907136e-07, |
| "loss": -0.0088, |
| "reward": -0.11346174217760563, |
| "reward_std": 0.24242284521460533, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2262607365846634, |
| "rewards/format_reward": 0.06250000186264515, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1215.3750457763672, |
| "epoch": 0.1353898886032562, |
| "grad_norm": 0.371949166059494, |
| "kl": 0.0008922815322875977, |
| "learning_rate": 9.908088623197048e-07, |
| "loss": -0.0848, |
| "reward": 0.8136582225561142, |
| "reward_std": 0.652880847454071, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.15616203658282757, |
| "rewards/format_reward": 0.4270833432674408, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1669.0625305175781, |
| "epoch": 0.13710368466152528, |
| "grad_norm": 0.26468425989151, |
| "kl": 6.92605972290039e-05, |
| "learning_rate": 9.901664203302124e-07, |
| "loss": 0.0218, |
| "reward": 0.18439925089478493, |
| "reward_std": 0.6725399419665337, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.22035246714949608, |
| "rewards/format_reward": 0.19791666883975267, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1511.5208435058594, |
| "epoch": 0.13881748071979436, |
| "grad_norm": 0.2657768726348877, |
| "kl": 9.560585021972656e-05, |
| "learning_rate": 9.895025252503755e-07, |
| "loss": 0.0318, |
| "reward": 0.5059427544474602, |
| "reward_std": 0.5254486501216888, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.18378128111362457, |
| "rewards/format_reward": 0.2395833358168602, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1469.8958740234375, |
| "epoch": 0.1405312767780634, |
| "grad_norm": 0.3196522891521454, |
| "kl": 0.00023925304412841797, |
| "learning_rate": 9.888172094375033e-07, |
| "loss": 0.0289, |
| "reward": 0.5352756977081299, |
| "reward_std": 0.9369603246450424, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.17629460990428925, |
| "rewards/format_reward": 0.2916666753590107, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1491.2708435058594, |
| "epoch": 0.14224507283633248, |
| "grad_norm": 0.2970998287200928, |
| "kl": 0.0001386404037475586, |
| "learning_rate": 9.881105062929221e-07, |
| "loss": 0.0842, |
| "reward": 0.5559089332818985, |
| "reward_std": 0.516811840236187, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.20787525177001953, |
| "rewards/format_reward": 0.21875000558793545, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1434.6666870117188, |
| "epoch": 0.14395886889460155, |
| "grad_norm": 0.2803492844104767, |
| "kl": 8.761882781982422e-05, |
| "learning_rate": 9.873824502603459e-07, |
| "loss": 0.0154, |
| "reward": 0.6808804646134377, |
| "reward_std": 0.6223056390881538, |
| "rewards/accuracy_reward": 0.11458333861082792, |
| "rewards/cosine_scaled_reward": -0.14863869547843933, |
| "rewards/format_reward": 0.2708333386108279, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1502.0417175292969, |
| "epoch": 0.1456726649528706, |
| "grad_norm": 0.2663383483886719, |
| "kl": 7.605552673339844e-05, |
| "learning_rate": 9.866330768241983e-07, |
| "loss": -0.0259, |
| "reward": 0.7719027325510979, |
| "reward_std": 0.6865327507257462, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/cosine_scaled_reward": -0.13260741718113422, |
| "rewards/format_reward": 0.2708333358168602, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1383.9583740234375, |
| "epoch": 0.14738646101113967, |
| "grad_norm": 0.26092609763145447, |
| "kl": 9.548664093017578e-05, |
| "learning_rate": 9.85862422507884e-07, |
| "loss": -0.0524, |
| "reward": 0.25671922299079597, |
| "reward_std": 0.6172372698783875, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.28932496905326843, |
| "rewards/format_reward": 0.34375000558793545, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1728.7291870117188, |
| "epoch": 0.14910025706940874, |
| "grad_norm": 0.24404188990592957, |
| "kl": 6.109476089477539e-05, |
| "learning_rate": 9.850705248720068e-07, |
| "loss": 0.0018, |
| "reward": -0.22325069084763527, |
| "reward_std": 0.5693457275629044, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.29550234228372574, |
| "rewards/format_reward": 0.07291666697710752, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1664.0833740234375, |
| "epoch": 0.15081405312767782, |
| "grad_norm": 0.29350191354751587, |
| "kl": 6.55055046081543e-05, |
| "learning_rate": 9.8425742251254e-07, |
| "loss": 0.0256, |
| "reward": 0.012098066508769989, |
| "reward_std": 0.5351692587137222, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2370474822819233, |
| "rewards/format_reward": 0.11458333488553762, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1432.4792175292969, |
| "epoch": 0.15252784918594686, |
| "grad_norm": 0.2622717022895813, |
| "kl": 0.00010013580322265625, |
| "learning_rate": 9.83423155058946e-07, |
| "loss": 0.0146, |
| "reward": 0.28211013600230217, |
| "reward_std": 0.6477071270346642, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.26215092837810516, |
| "rewards/format_reward": 0.30208334513008595, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1464.2291870117188, |
| "epoch": 0.15424164524421594, |
| "grad_norm": 0.33814817667007446, |
| "kl": 0.00013047456741333008, |
| "learning_rate": 9.825677631722435e-07, |
| "loss": -0.0532, |
| "reward": 0.2572305239737034, |
| "reward_std": 0.387471754103899, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.22083472087979317, |
| "rewards/format_reward": 0.17708333861082792, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1426.6458435058594, |
| "epoch": 0.155955441302485, |
| "grad_norm": 0.2609995901584625, |
| "kl": 0.0002288818359375, |
| "learning_rate": 9.816912885430258e-07, |
| "loss": 0.0247, |
| "reward": 0.14339020662009716, |
| "reward_std": 0.5801211446523666, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2412719689309597, |
| "rewards/format_reward": 0.22916666697710752, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1398.1250610351562, |
| "epoch": 0.15766923736075408, |
| "grad_norm": 0.2580174505710602, |
| "kl": 7.218122482299805e-05, |
| "learning_rate": 9.807937738894303e-07, |
| "loss": 0.0258, |
| "reward": 0.5541771352291107, |
| "reward_std": 0.5360789932310581, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.20377103984355927, |
| "rewards/format_reward": 0.26041667349636555, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1538.2708740234375, |
| "epoch": 0.15938303341902313, |
| "grad_norm": 0.25805678963661194, |
| "kl": 8.285045623779297e-05, |
| "learning_rate": 9.798752629550546e-07, |
| "loss": -0.0287, |
| "reward": 0.5447847582399845, |
| "reward_std": 0.9511096179485321, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.21756825502961874, |
| "rewards/format_reward": 0.2812500111758709, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1539.3958740234375, |
| "epoch": 0.1610968294772922, |
| "grad_norm": 0.33063074946403503, |
| "kl": 0.0005064010620117188, |
| "learning_rate": 9.78935800506826e-07, |
| "loss": 0.0871, |
| "reward": 0.9277739748358727, |
| "reward_std": 0.7628872096538544, |
| "rewards/accuracy_reward": 0.09375000186264515, |
| "rewards/cosine_scaled_reward": -0.15467804670333862, |
| "rewards/format_reward": 0.37500001303851604, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1465.6041870117188, |
| "epoch": 0.16281062553556128, |
| "grad_norm": 0.2793846130371094, |
| "kl": 0.00011479854583740234, |
| "learning_rate": 9.779754323328192e-07, |
| "loss": 0.0828, |
| "reward": 0.27345613669604063, |
| "reward_std": 0.5890941619873047, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.23359986767172813, |
| "rewards/format_reward": 0.22916666977107525, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1562.0833740234375, |
| "epoch": 0.16452442159383032, |
| "grad_norm": 0.2997168004512787, |
| "kl": 0.00010025501251220703, |
| "learning_rate": 9.769942052400235e-07, |
| "loss": -0.0651, |
| "reward": 0.11648451164364815, |
| "reward_std": 0.7505641058087349, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.27028296515345573, |
| "rewards/format_reward": 0.18750000558793545, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1477.6666870117188, |
| "epoch": 0.1662382176520994, |
| "grad_norm": 0.3399166762828827, |
| "kl": 0.0001264810562133789, |
| "learning_rate": 9.759921670520634e-07, |
| "loss": 0.0139, |
| "reward": 0.28585558384656906, |
| "reward_std": 0.4958939217031002, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.21860605105757713, |
| "rewards/format_reward": 0.3125000074505806, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1550.0000305175781, |
| "epoch": 0.16795201371036847, |
| "grad_norm": 0.2669260799884796, |
| "kl": 0.00010633468627929688, |
| "learning_rate": 9.749693666068663e-07, |
| "loss": 0.0197, |
| "reward": 0.5891378601081669, |
| "reward_std": 0.794895127415657, |
| "rewards/accuracy_reward": 0.0520833358168602, |
| "rewards/cosine_scaled_reward": -0.24003576394170523, |
| "rewards/format_reward": 0.375, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1576.2083740234375, |
| "epoch": 0.16966580976863754, |
| "grad_norm": 0.2624787986278534, |
| "kl": 0.0001227855682373047, |
| "learning_rate": 9.739258537542835e-07, |
| "loss": -0.0209, |
| "reward": 0.16898822411894798, |
| "reward_std": 0.6391639858484268, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2093992941081524, |
| "rewards/format_reward": 0.1562500037252903, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1460.6875305175781, |
| "epoch": 0.1713796058269066, |
| "grad_norm": 0.34687182307243347, |
| "kl": 0.0001291036605834961, |
| "learning_rate": 9.728616793536587e-07, |
| "loss": -0.0869, |
| "reward": 0.3641693815588951, |
| "reward_std": 0.7006602585315704, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.2502584457397461, |
| "rewards/format_reward": 0.32291666977107525, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1485.9792175292969, |
| "epoch": 0.17309340188517566, |
| "grad_norm": 0.2527833580970764, |
| "kl": 9.036064147949219e-05, |
| "learning_rate": 9.717768952713511e-07, |
| "loss": 0.0023, |
| "reward": 0.4851684862514958, |
| "reward_std": 0.5674109682440758, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.20687074586749077, |
| "rewards/format_reward": 0.2500000074505806, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1580.2708740234375, |
| "epoch": 0.17480719794344474, |
| "grad_norm": 0.28545081615448, |
| "kl": 9.453296661376953e-05, |
| "learning_rate": 9.706715543782064e-07, |
| "loss": 0.0331, |
| "reward": 0.4691831008531153, |
| "reward_std": 0.6173286661505699, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.18831564113497734, |
| "rewards/format_reward": 0.25000000558793545, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1568.5625305175781, |
| "epoch": 0.17652099400171378, |
| "grad_norm": 0.24966423213481903, |
| "kl": 0.0001354813575744629, |
| "learning_rate": 9.695457105469804e-07, |
| "loss": 0.0499, |
| "reward": 0.6344753429293633, |
| "reward_std": 0.8610301837325096, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.1816388964653015, |
| "rewards/format_reward": 0.20833334419876337, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1620.4583435058594, |
| "epoch": 0.17823479005998286, |
| "grad_norm": 0.2664543390274048, |
| "kl": 0.00010859966278076172, |
| "learning_rate": 9.683994186497132e-07, |
| "loss": 0.018, |
| "reward": 0.17410985194146633, |
| "reward_std": 0.44621556997299194, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.18129736836999655, |
| "rewards/format_reward": 0.09375000186264515, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1476.4791870117188, |
| "epoch": 0.17994858611825193, |
| "grad_norm": 0.30209964513778687, |
| "kl": 0.00010883808135986328, |
| "learning_rate": 9.672327345550543e-07, |
| "loss": -0.0613, |
| "reward": 0.41774739080574363, |
| "reward_std": 0.7054343000054359, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2176332138478756, |
| "rewards/format_reward": 0.354166679084301, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1575.7083740234375, |
| "epoch": 0.181662382176521, |
| "grad_norm": 0.305949866771698, |
| "kl": 8.207559585571289e-05, |
| "learning_rate": 9.66045715125541e-07, |
| "loss": -0.0178, |
| "reward": 0.19928239285945892, |
| "reward_std": 0.5839960649609566, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.24334679171442986, |
| "rewards/format_reward": 0.22916667442768812, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1595.4375305175781, |
| "epoch": 0.18337617823479005, |
| "grad_norm": 0.25899356603622437, |
| "kl": 0.00011038780212402344, |
| "learning_rate": 9.648384182148252e-07, |
| "loss": -0.0196, |
| "reward": 0.3261868259869516, |
| "reward_std": 0.7823206260800362, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.2219230905175209, |
| "rewards/format_reward": 0.2291666753590107, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1706.9166870117188, |
| "epoch": 0.18508997429305912, |
| "grad_norm": 0.24813760817050934, |
| "kl": 6.186962127685547e-05, |
| "learning_rate": 9.636109026648554e-07, |
| "loss": -0.0167, |
| "reward": 0.09848582930862904, |
| "reward_std": 0.49404577910900116, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.2548905946314335, |
| "rewards/format_reward": 0.1250000037252903, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1367.3333435058594, |
| "epoch": 0.1868037703513282, |
| "grad_norm": 0.310843825340271, |
| "kl": 0.00025588274002075195, |
| "learning_rate": 9.623632283030077e-07, |
| "loss": 0.0524, |
| "reward": 0.2563808672130108, |
| "reward_std": 0.5945833437144756, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.1962066236883402, |
| "rewards/format_reward": 0.21875001024454832, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1545.8542175292969, |
| "epoch": 0.18851756640959727, |
| "grad_norm": 0.3281257748603821, |
| "kl": 7.37905502319336e-05, |
| "learning_rate": 9.610954559391704e-07, |
| "loss": -0.0966, |
| "reward": 0.5111359301954508, |
| "reward_std": 1.0251065343618393, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.23871351778507233, |
| "rewards/format_reward": 0.2604166744276881, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1352.9791870117188, |
| "epoch": 0.19023136246786632, |
| "grad_norm": 0.29662254452705383, |
| "kl": 8.809566497802734e-05, |
| "learning_rate": 9.598076473627796e-07, |
| "loss": 0.0491, |
| "reward": 0.5007766149938107, |
| "reward_std": 0.5933823436498642, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.219840994104743, |
| "rewards/format_reward": 0.3020833469927311, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1521.0833740234375, |
| "epoch": 0.1919451585261354, |
| "grad_norm": 0.22664791345596313, |
| "kl": 9.524822235107422e-05, |
| "learning_rate": 9.58499865339809e-07, |
| "loss": 0.0133, |
| "reward": 0.9934604372829199, |
| "reward_std": 0.8234051987528801, |
| "rewards/accuracy_reward": 0.09375, |
| "rewards/cosine_scaled_reward": -0.12148096412420273, |
| "rewards/format_reward": 0.3645833358168602, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1488.0000610351562, |
| "epoch": 0.19365895458440446, |
| "grad_norm": 0.26795610785484314, |
| "kl": 0.00011110305786132812, |
| "learning_rate": 9.571721736097088e-07, |
| "loss": -0.0702, |
| "reward": 0.3580288141965866, |
| "reward_std": 0.5283620953559875, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.22576149180531502, |
| "rewards/format_reward": 0.2395833358168602, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1324.2916870117188, |
| "epoch": 0.1953727506426735, |
| "grad_norm": 0.2916397452354431, |
| "kl": 0.00011074542999267578, |
| "learning_rate": 9.55824636882301e-07, |
| "loss": -0.0276, |
| "reward": 0.9822416566312313, |
| "reward_std": 0.8637243807315826, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/cosine_scaled_reward": -0.17669491842389107, |
| "rewards/format_reward": 0.447916679084301, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1271.9166870117188, |
| "epoch": 0.19708654670094258, |
| "grad_norm": 0.36266040802001953, |
| "kl": 0.00022935867309570312, |
| "learning_rate": 9.54457320834625e-07, |
| "loss": 0.0726, |
| "reward": 1.2695526629686356, |
| "reward_std": 1.0137326195836067, |
| "rewards/accuracy_reward": 0.13541666977107525, |
| "rewards/cosine_scaled_reward": -0.16991223488003016, |
| "rewards/format_reward": 0.489583358168602, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1590.9791870117188, |
| "epoch": 0.19880034275921166, |
| "grad_norm": 0.258552223443985, |
| "kl": 0.0002028942108154297, |
| "learning_rate": 9.530702921077358e-07, |
| "loss": 0.0004, |
| "reward": 0.2832749206572771, |
| "reward_std": 0.37558774650096893, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.23772768676280975, |
| "rewards/format_reward": 0.21875000651925802, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1381.0000610351562, |
| "epoch": 0.20051413881748073, |
| "grad_norm": 0.30766648054122925, |
| "kl": 0.0001800060272216797, |
| "learning_rate": 9.516636183034564e-07, |
| "loss": 0.0235, |
| "reward": 0.7005061060190201, |
| "reward_std": 0.545831985771656, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.10087039694190025, |
| "rewards/format_reward": 0.30208334140479565, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1369.0208740234375, |
| "epoch": 0.20222793487574978, |
| "grad_norm": 0.2912021577358246, |
| "kl": 0.00012183189392089844, |
| "learning_rate": 9.502373679810839e-07, |
| "loss": 0.0264, |
| "reward": 0.8078554645180702, |
| "reward_std": 0.9160244166851044, |
| "rewards/accuracy_reward": 0.07291666977107525, |
| "rewards/cosine_scaled_reward": -0.1643010601401329, |
| "rewards/format_reward": 0.3854166716337204, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1497.5833740234375, |
| "epoch": 0.20394173093401885, |
| "grad_norm": 0.3184635043144226, |
| "kl": 7.218122482299805e-05, |
| "learning_rate": 9.487916106540465e-07, |
| "loss": 0.0635, |
| "reward": 0.756352175027132, |
| "reward_std": 0.9685456156730652, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.202363814227283, |
| "rewards/format_reward": 0.354166679084301, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1514.4583740234375, |
| "epoch": 0.20565552699228792, |
| "grad_norm": 0.301592618227005, |
| "kl": 7.611513137817383e-05, |
| "learning_rate": 9.473264167865171e-07, |
| "loss": 0.0527, |
| "reward": 0.6178061664104462, |
| "reward_std": 1.1512691602110863, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/cosine_scaled_reward": -0.16802122257649899, |
| "rewards/format_reward": 0.26041666977107525, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1497.7500610351562, |
| "epoch": 0.207369323050557, |
| "grad_norm": 0.2697899639606476, |
| "kl": 0.00011616945266723633, |
| "learning_rate": 9.458418577899774e-07, |
| "loss": -0.0769, |
| "reward": 0.5254677496850491, |
| "reward_std": 0.8822471648454666, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.17704324051737785, |
| "rewards/format_reward": 0.23958334140479565, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1671.2083740234375, |
| "epoch": 0.20908311910882604, |
| "grad_norm": 0.2589820623397827, |
| "kl": 8.726119995117188e-05, |
| "learning_rate": 9.443380060197385e-07, |
| "loss": 0.0032, |
| "reward": 0.12582964450120926, |
| "reward_std": 0.6176033914089203, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.24815166369080544, |
| "rewards/format_reward": 0.2187500037252903, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1429.4166870117188, |
| "epoch": 0.21079691516709512, |
| "grad_norm": 0.3138475716114044, |
| "kl": 0.0001118779182434082, |
| "learning_rate": 9.428149347714143e-07, |
| "loss": 0.0055, |
| "reward": 0.5503488332033157, |
| "reward_std": 1.1257527768611908, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.19736815616488457, |
| "rewards/format_reward": 0.23958333861082792, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1601.3333740234375, |
| "epoch": 0.2125107112253642, |
| "grad_norm": 0.305877685546875, |
| "kl": 8.064508438110352e-05, |
| "learning_rate": 9.412727182773486e-07, |
| "loss": -0.0732, |
| "reward": 0.2043483555316925, |
| "reward_std": 0.25610289350152016, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.21675095334649086, |
| "rewards/format_reward": 0.1458333395421505, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1594.25, |
| "epoch": 0.21422450728363324, |
| "grad_norm": 0.32135531306266785, |
| "kl": 9.369850158691406e-05, |
| "learning_rate": 9.397114317029974e-07, |
| "loss": -0.0312, |
| "reward": 0.7128554601222277, |
| "reward_std": 0.7205516248941422, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.1431606486439705, |
| "rewards/format_reward": 0.2083333358168602, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1565.0000305175781, |
| "epoch": 0.2159383033419023, |
| "grad_norm": 0.30996406078338623, |
| "kl": 0.00023823976516723633, |
| "learning_rate": 9.381311511432658e-07, |
| "loss": -0.057, |
| "reward": 0.22010673861950636, |
| "reward_std": 0.7089016139507294, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.23909669369459152, |
| "rewards/format_reward": 0.26041667722165585, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1514.2708740234375, |
| "epoch": 0.21765209940017138, |
| "grad_norm": 0.2887818515300751, |
| "kl": 0.00017833709716796875, |
| "learning_rate": 9.36531953618799e-07, |
| "loss": -0.0256, |
| "reward": 0.40598735213279724, |
| "reward_std": 0.668736569583416, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.21378641575574875, |
| "rewards/format_reward": 0.1979166716337204, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1636.3958740234375, |
| "epoch": 0.21936589545844046, |
| "grad_norm": 0.3046552836894989, |
| "kl": 0.0016319751739501953, |
| "learning_rate": 9.34913917072228e-07, |
| "loss": -0.0277, |
| "reward": 0.14084632135927677, |
| "reward_std": 0.5165095031261444, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.265597328543663, |
| "rewards/format_reward": 0.17708333488553762, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1377.1666870117188, |
| "epoch": 0.2210796915167095, |
| "grad_norm": 0.30127936601638794, |
| "kl": 0.00021696090698242188, |
| "learning_rate": 9.332771203643714e-07, |
| "loss": 0.0466, |
| "reward": 0.6318524926900864, |
| "reward_std": 0.8379577845335007, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.20013105496764183, |
| "rewards/format_reward": 0.3437500074505806, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1281.7917175292969, |
| "epoch": 0.22279348757497858, |
| "grad_norm": 0.33132994174957275, |
| "kl": 0.00022983551025390625, |
| "learning_rate": 9.316216432703916e-07, |
| "loss": -0.0199, |
| "reward": 1.4362520650029182, |
| "reward_std": 0.7704244181513786, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/cosine_scaled_reward": -0.1344580352306366, |
| "rewards/format_reward": 0.4166666818782687, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1669.8125610351562, |
| "epoch": 0.22450728363324765, |
| "grad_norm": 0.26186782121658325, |
| "kl": 0.00011074542999267578, |
| "learning_rate": 9.299475664759068e-07, |
| "loss": -0.0367, |
| "reward": 0.0669812560081482, |
| "reward_std": 0.6163998395204544, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2444169856607914, |
| "rewards/format_reward": 0.17708333674818277, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1598.8541870117188, |
| "epoch": 0.2262210796915167, |
| "grad_norm": 0.27530592679977417, |
| "kl": 8.606910705566406e-05, |
| "learning_rate": 9.282549715730579e-07, |
| "loss": -0.022, |
| "reward": 0.4553011879324913, |
| "reward_std": 0.7698710784316063, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.19618215784430504, |
| "rewards/format_reward": 0.2812500111758709, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1476.7916870117188, |
| "epoch": 0.22793487574978577, |
| "grad_norm": 0.31282302737236023, |
| "kl": 0.00011754035949707031, |
| "learning_rate": 9.265439410565328e-07, |
| "loss": 0.0149, |
| "reward": 0.5651843696832657, |
| "reward_std": 0.8456990644335747, |
| "rewards/accuracy_reward": 0.07291666977107525, |
| "rewards/cosine_scaled_reward": -0.21052773855626583, |
| "rewards/format_reward": 0.2291666716337204, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1257.2500457763672, |
| "epoch": 0.22964867180805484, |
| "grad_norm": 0.3299978971481323, |
| "kl": 0.0005350112915039062, |
| "learning_rate": 9.248145583195447e-07, |
| "loss": -0.0553, |
| "reward": 0.9383534267544746, |
| "reward_std": 0.8029176890850067, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.1500500962138176, |
| "rewards/format_reward": 0.4375000111758709, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1279.9583740234375, |
| "epoch": 0.23136246786632392, |
| "grad_norm": 0.26606979966163635, |
| "kl": 0.0002206563949584961, |
| "learning_rate": 9.230669076497687e-07, |
| "loss": -0.064, |
| "reward": 0.8971398249268532, |
| "reward_std": 0.8118537589907646, |
| "rewards/accuracy_reward": 0.09375000279396772, |
| "rewards/cosine_scaled_reward": -0.1715338509529829, |
| "rewards/format_reward": 0.3437500111758709, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1403.9167175292969, |
| "epoch": 0.23307626392459296, |
| "grad_norm": 0.33854958415031433, |
| "kl": 0.0002644062042236328, |
| "learning_rate": 9.213010742252327e-07, |
| "loss": 0.0447, |
| "reward": 0.45938077941536903, |
| "reward_std": 0.87321837246418, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.2194330394268036, |
| "rewards/format_reward": 0.2395833395421505, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1600.7291870117188, |
| "epoch": 0.23479005998286204, |
| "grad_norm": 0.2964058220386505, |
| "kl": 7.575750350952148e-05, |
| "learning_rate": 9.195171441101668e-07, |
| "loss": 0.0798, |
| "reward": -0.00549701415002346, |
| "reward_std": 0.4839354231953621, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.26485851779580116, |
| "rewards/format_reward": 0.197916679084301, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1569.2500305175781, |
| "epoch": 0.2365038560411311, |
| "grad_norm": 0.27017030119895935, |
| "kl": 6.553530693054199e-05, |
| "learning_rate": 9.177152042508077e-07, |
| "loss": 0.0473, |
| "reward": 0.36223573237657547, |
| "reward_std": 0.5121153928339481, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.2260081209242344, |
| "rewards/format_reward": 0.1770833395421505, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1500.1250305175781, |
| "epoch": 0.23821765209940018, |
| "grad_norm": 0.3094533681869507, |
| "kl": 0.00014731287956237793, |
| "learning_rate": 9.158953424711624e-07, |
| "loss": -0.0638, |
| "reward": 0.3372825998812914, |
| "reward_std": 0.35855112969875336, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2001120790373534, |
| "rewards/format_reward": 0.19791667256504297, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1184.2708587646484, |
| "epoch": 0.23993144815766923, |
| "grad_norm": 0.35570886731147766, |
| "kl": 0.00015413761138916016, |
| "learning_rate": 9.140576474687263e-07, |
| "loss": -0.0027, |
| "reward": 0.796763252466917, |
| "reward_std": 0.8294687867164612, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.18409650027751923, |
| "rewards/format_reward": 0.4375000149011612, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1546.2917175292969, |
| "epoch": 0.2416452442159383, |
| "grad_norm": 0.3957357406616211, |
| "kl": 0.0001176595687866211, |
| "learning_rate": 9.122022088101613e-07, |
| "loss": 0.0978, |
| "reward": 0.26207078620791435, |
| "reward_std": 0.5963894873857498, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.24103039875626564, |
| "rewards/format_reward": 0.22916667070239782, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1646.3750305175781, |
| "epoch": 0.24335904027420738, |
| "grad_norm": 0.29940950870513916, |
| "kl": 0.00010442733764648438, |
| "learning_rate": 9.103291169269299e-07, |
| "loss": 0.0115, |
| "reward": 0.12744126096367836, |
| "reward_std": 0.7610968500375748, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.2749045342206955, |
| "rewards/format_reward": 0.16666667070239782, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1623.1250610351562, |
| "epoch": 0.24507283633247642, |
| "grad_norm": 0.23946928977966309, |
| "kl": 0.00012135505676269531, |
| "learning_rate": 9.084384631108882e-07, |
| "loss": -0.0226, |
| "reward": 0.05879242718219757, |
| "reward_std": 0.7065651714801788, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2815237008035183, |
| "rewards/format_reward": 0.2291666679084301, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1717.0417175292969, |
| "epoch": 0.2467866323907455, |
| "grad_norm": 0.30980589985847473, |
| "kl": 5.7578086853027344e-05, |
| "learning_rate": 9.065303395098358e-07, |
| "loss": 0.0031, |
| "reward": -0.03822072362527251, |
| "reward_std": 0.41779977828264236, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.27143828570842743, |
| "rewards/format_reward": 0.17708334047347307, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1459.7083740234375, |
| "epoch": 0.24850042844901457, |
| "grad_norm": 0.26551711559295654, |
| "kl": 9.238719940185547e-05, |
| "learning_rate": 9.046048391230247e-07, |
| "loss": 0.0405, |
| "reward": 0.9580619968473911, |
| "reward_std": 0.65601646900177, |
| "rewards/accuracy_reward": 0.1145833358168602, |
| "rewards/cosine_scaled_reward": -0.1509431917220354, |
| "rewards/format_reward": 0.27083334140479565, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1350.8333740234375, |
| "epoch": 0.25021422450728364, |
| "grad_norm": 0.22373563051223755, |
| "kl": 0.0002359151840209961, |
| "learning_rate": 9.026620557966279e-07, |
| "loss": -0.0072, |
| "reward": 0.5567201105877757, |
| "reward_std": 0.7935501113533974, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.20699115842580795, |
| "rewards/format_reward": 0.3020833358168602, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1449.2291870117188, |
| "epoch": 0.2519280205655527, |
| "grad_norm": 0.24441158771514893, |
| "kl": 0.00011551380157470703, |
| "learning_rate": 9.007020842191634e-07, |
| "loss": -0.0188, |
| "reward": 0.7537737488746643, |
| "reward_std": 0.6216295659542084, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.1462303502485156, |
| "rewards/format_reward": 0.3229166716337204, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1600.7084045410156, |
| "epoch": 0.2536418166238218, |
| "grad_norm": 0.26157715916633606, |
| "kl": 8.0108642578125e-05, |
| "learning_rate": 8.987250199168808e-07, |
| "loss": -0.005, |
| "reward": 0.2785342447459698, |
| "reward_std": 0.6226839050650597, |
| "rewards/accuracy_reward": 0.0520833358168602, |
| "rewards/cosine_scaled_reward": -0.2609979063272476, |
| "rewards/format_reward": 0.17708333767950535, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1360.8750305175781, |
| "epoch": 0.25535561268209084, |
| "grad_norm": 0.2786451578140259, |
| "kl": 0.000179290771484375, |
| "learning_rate": 8.967309592491052e-07, |
| "loss": -0.0155, |
| "reward": 0.3341223709285259, |
| "reward_std": 0.6954998485744, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.21542707458138466, |
| "rewards/format_reward": 0.3229166753590107, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1405.9166870117188, |
| "epoch": 0.2570694087403599, |
| "grad_norm": 0.22569335997104645, |
| "kl": 0.0001271963119506836, |
| "learning_rate": 8.9471999940354e-07, |
| "loss": 0.0332, |
| "reward": 1.1879419684410095, |
| "reward_std": 0.849453404545784, |
| "rewards/accuracy_reward": 0.15625000465661287, |
| "rewards/cosine_scaled_reward": -0.07610182277858257, |
| "rewards/format_reward": 0.3645833469927311, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1398.062515258789, |
| "epoch": 0.258783204798629, |
| "grad_norm": 0.2917872667312622, |
| "kl": 0.0003751516342163086, |
| "learning_rate": 8.926922383915315e-07, |
| "loss": 0.0708, |
| "reward": 0.2504497095942497, |
| "reward_std": 0.7372590154409409, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.29317696392536163, |
| "rewards/format_reward": 0.2812500139698386, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1530.8958740234375, |
| "epoch": 0.26049700085689803, |
| "grad_norm": 0.29970836639404297, |
| "kl": 0.00013697147369384766, |
| "learning_rate": 8.906477750432903e-07, |
| "loss": 0.0914, |
| "reward": 0.3063769284635782, |
| "reward_std": 0.6683933213353157, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.22888047248125076, |
| "rewards/format_reward": 0.2083333395421505, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1557.4583740234375, |
| "epoch": 0.2622107969151671, |
| "grad_norm": 0.31202468276023865, |
| "kl": 0.00016236305236816406, |
| "learning_rate": 8.88586709003076e-07, |
| "loss": -0.0161, |
| "reward": 0.23547574784606695, |
| "reward_std": 0.623480673879385, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.212197445333004, |
| "rewards/format_reward": 0.2395833432674408, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1524.0000305175781, |
| "epoch": 0.2639245929734362, |
| "grad_norm": 0.2832154631614685, |
| "kl": 0.00022339820861816406, |
| "learning_rate": 8.865091407243394e-07, |
| "loss": 0.0183, |
| "reward": 0.8646783344447613, |
| "reward_std": 0.8572295308113098, |
| "rewards/accuracy_reward": 0.09375000186264515, |
| "rewards/cosine_scaled_reward": -0.16477943025529385, |
| "rewards/format_reward": 0.26041667722165585, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1595.1041870117188, |
| "epoch": 0.2656383890317052, |
| "grad_norm": 0.25693395733833313, |
| "kl": 0.00016927719116210938, |
| "learning_rate": 8.844151714648274e-07, |
| "loss": -0.023, |
| "reward": -0.12136609526351094, |
| "reward_std": 0.3815436065196991, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.26241349801421165, |
| "rewards/format_reward": 0.13541667070239782, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1576.6250610351562, |
| "epoch": 0.26735218508997427, |
| "grad_norm": 0.25725266337394714, |
| "kl": 0.00014984607696533203, |
| "learning_rate": 8.823049032816478e-07, |
| "loss": 0.0012, |
| "reward": 0.24715298786759377, |
| "reward_std": 0.7283157333731651, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2703519072383642, |
| "rewards/format_reward": 0.27083334140479565, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1352.2917022705078, |
| "epoch": 0.26906598114824337, |
| "grad_norm": 0.3138567805290222, |
| "kl": 0.00035691261291503906, |
| "learning_rate": 8.801784390262943e-07, |
| "loss": 0.1456, |
| "reward": 0.3800088334828615, |
| "reward_std": 0.6559219062328339, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.2365667223930359, |
| "rewards/format_reward": 0.20833334233611822, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1562.9375305175781, |
| "epoch": 0.2707797772065124, |
| "grad_norm": 0.34799516201019287, |
| "kl": 0.0002155303955078125, |
| "learning_rate": 8.780358823396352e-07, |
| "loss": 0.028, |
| "reward": 0.3663053174968809, |
| "reward_std": 0.5830123052001, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.18629762902855873, |
| "rewards/format_reward": 0.19791667256504297, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1593.7708740234375, |
| "epoch": 0.27249357326478146, |
| "grad_norm": 0.2532951831817627, |
| "kl": 0.00032138824462890625, |
| "learning_rate": 8.758773376468604e-07, |
| "loss": 0.0147, |
| "reward": 0.010641081258654594, |
| "reward_std": 0.6984475031495094, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2668936438858509, |
| "rewards/format_reward": 0.15625000651925802, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1554.9583435058594, |
| "epoch": 0.27420736932305056, |
| "grad_norm": 0.24631713330745697, |
| "kl": 0.0002465248107910156, |
| "learning_rate": 8.737029101523929e-07, |
| "loss": -0.0377, |
| "reward": 0.3181496039032936, |
| "reward_std": 0.3356131613254547, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.13599055260419846, |
| "rewards/format_reward": 0.15625, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1536.9375305175781, |
| "epoch": 0.2759211653813196, |
| "grad_norm": 0.29518961906433105, |
| "kl": 0.0003285408020019531, |
| "learning_rate": 8.715127058347614e-07, |
| "loss": -0.0098, |
| "reward": 0.450228420086205, |
| "reward_std": 0.8069275915622711, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.20325740799307823, |
| "rewards/format_reward": 0.2083333395421505, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1397.5625305175781, |
| "epoch": 0.2776349614395887, |
| "grad_norm": 0.32521313428878784, |
| "kl": 0.0004208087921142578, |
| "learning_rate": 8.693068314414344e-07, |
| "loss": -0.0538, |
| "reward": 0.2773311994969845, |
| "reward_std": 0.5018344074487686, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2300034947693348, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1482.354248046875, |
| "epoch": 0.27934875749785776, |
| "grad_norm": 0.26695311069488525, |
| "kl": 0.0003781318664550781, |
| "learning_rate": 8.670853944836176e-07, |
| "loss": 0.0127, |
| "reward": 0.5796077568084002, |
| "reward_std": 0.947176143527031, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.21064921002835035, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1325.2083740234375, |
| "epoch": 0.2810625535561268, |
| "grad_norm": 0.29990634322166443, |
| "kl": 0.0003497600555419922, |
| "learning_rate": 8.648485032310144e-07, |
| "loss": -0.0518, |
| "reward": 0.8711699396371841, |
| "reward_std": 0.6968246772885323, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.18954484723508358, |
| "rewards/format_reward": 0.3750000186264515, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1571.4375305175781, |
| "epoch": 0.2827763496143959, |
| "grad_norm": 0.2641393542289734, |
| "kl": 0.00020003318786621094, |
| "learning_rate": 8.625962667065487e-07, |
| "loss": 0.0589, |
| "reward": 0.3537735566496849, |
| "reward_std": 0.6288779303431511, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.20627648755908012, |
| "rewards/format_reward": 0.22916667722165585, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1436.1458740234375, |
| "epoch": 0.28449014567266495, |
| "grad_norm": 0.3507601320743561, |
| "kl": 0.0003981590270996094, |
| "learning_rate": 8.603287946810513e-07, |
| "loss": 0.0933, |
| "reward": 0.4801969490945339, |
| "reward_std": 0.6994242109358311, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.18304483219981194, |
| "rewards/format_reward": 0.2708333469927311, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1492.7500305175781, |
| "epoch": 0.286203941730934, |
| "grad_norm": 0.26833590865135193, |
| "kl": 0.000438690185546875, |
| "learning_rate": 8.580461976679099e-07, |
| "loss": -0.0162, |
| "reward": 0.6301786154508591, |
| "reward_std": 0.9764036163687706, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.15223283600062132, |
| "rewards/format_reward": 0.29166667722165585, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1572.6458740234375, |
| "epoch": 0.2879177377892031, |
| "grad_norm": 0.25694307684898376, |
| "kl": 0.0003559589385986328, |
| "learning_rate": 8.557485869176825e-07, |
| "loss": 0.0152, |
| "reward": 0.27175838500261307, |
| "reward_std": 0.6918256878852844, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.221031554043293, |
| "rewards/format_reward": 0.1979166716337204, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1476.6666870117188, |
| "epoch": 0.28963153384747214, |
| "grad_norm": 0.3062116801738739, |
| "kl": 0.0006361007690429688, |
| "learning_rate": 8.534360744126753e-07, |
| "loss": -0.1259, |
| "reward": 0.41318080946803093, |
| "reward_std": 0.4180409014225006, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.1710204929113388, |
| "rewards/format_reward": 0.19791667256504297, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1407.8958587646484, |
| "epoch": 0.2913453299057412, |
| "grad_norm": 0.3516060411930084, |
| "kl": 0.0011429786682128906, |
| "learning_rate": 8.511087728614862e-07, |
| "loss": 0.0441, |
| "reward": 0.9393245745450258, |
| "reward_std": 0.6329760327935219, |
| "rewards/accuracy_reward": 0.1145833358168602, |
| "rewards/cosine_scaled_reward": -0.15719182137399912, |
| "rewards/format_reward": 0.2708333432674408, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1431.0208740234375, |
| "epoch": 0.2930591259640103, |
| "grad_norm": 0.3069850206375122, |
| "kl": 0.0004830360412597656, |
| "learning_rate": 8.487667956935087e-07, |
| "loss": 0.0076, |
| "reward": 0.7211491465568542, |
| "reward_std": 0.9154210761189461, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/cosine_scaled_reward": -0.18316034972667694, |
| "rewards/format_reward": 0.3645833507180214, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1627.6041870117188, |
| "epoch": 0.29477292202227934, |
| "grad_norm": 0.24381308257579803, |
| "kl": 0.00026917457580566406, |
| "learning_rate": 8.464102570534061e-07, |
| "loss": -0.0342, |
| "reward": 0.39864223077893257, |
| "reward_std": 0.8442265018820763, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.22351143136620522, |
| "rewards/format_reward": 0.1979166679084301, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1377.0416870117188, |
| "epoch": 0.29648671808054844, |
| "grad_norm": 0.2742233872413635, |
| "kl": 0.0032143592834472656, |
| "learning_rate": 8.440392717955475e-07, |
| "loss": 0.0075, |
| "reward": 0.43526358902454376, |
| "reward_std": 0.5948851481080055, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.23395783826708794, |
| "rewards/format_reward": 0.322916679084301, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1395.9167175292969, |
| "epoch": 0.2982005141388175, |
| "grad_norm": 0.48359552025794983, |
| "kl": 0.0013885498046875, |
| "learning_rate": 8.416539554784089e-07, |
| "loss": 0.0137, |
| "reward": 0.2430977914482355, |
| "reward_std": 0.5702231302857399, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.20221445709466934, |
| "rewards/format_reward": 0.2604166679084301, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1477.0208740234375, |
| "epoch": 0.29991431019708653, |
| "grad_norm": 0.34349390864372253, |
| "kl": 0.000823974609375, |
| "learning_rate": 8.392544243589427e-07, |
| "loss": 0.1064, |
| "reward": 0.09682288765907288, |
| "reward_std": 0.3903738856315613, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2545960769057274, |
| "rewards/format_reward": 0.2708333358168602, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1413.6666870117188, |
| "epoch": 0.30162810625535563, |
| "grad_norm": 0.4631779193878174, |
| "kl": 0.0005354881286621094, |
| "learning_rate": 8.368407953869103e-07, |
| "loss": -0.1349, |
| "reward": 0.6249970365315676, |
| "reward_std": 1.1163012832403183, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.1966523379087448, |
| "rewards/format_reward": 0.3750000176951289, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1708.0625305175781, |
| "epoch": 0.3033419023136247, |
| "grad_norm": 0.2329261600971222, |
| "kl": 0.00026154518127441406, |
| "learning_rate": 8.344131861991828e-07, |
| "loss": -0.0058, |
| "reward": 0.36284567788243294, |
| "reward_std": 0.4691716283559799, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.21771202981472015, |
| "rewards/format_reward": 0.1979166753590107, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1524.9583435058594, |
| "epoch": 0.3050556983718937, |
| "grad_norm": 0.28604593873023987, |
| "kl": 0.0003364086151123047, |
| "learning_rate": 8.319717151140072e-07, |
| "loss": -0.0166, |
| "reward": 0.6543274968862534, |
| "reward_std": 0.9791368544101715, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.17932307720184326, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1677.9583435058594, |
| "epoch": 0.3067694944301628, |
| "grad_norm": 0.30129143595695496, |
| "kl": 0.0004076957702636719, |
| "learning_rate": 8.295165011252396e-07, |
| "loss": 0.007, |
| "reward": 0.243562500923872, |
| "reward_std": 0.7496606484055519, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.219083484262228, |
| "rewards/format_reward": 0.18750000651925802, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1274.062515258789, |
| "epoch": 0.30848329048843187, |
| "grad_norm": 0.3613395690917969, |
| "kl": 0.0006461143493652344, |
| "learning_rate": 8.270476638965461e-07, |
| "loss": -0.0415, |
| "reward": 0.8658371195197105, |
| "reward_std": 1.10177643597126, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.1902889758348465, |
| "rewards/format_reward": 0.447916679084301, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1594.0000305175781, |
| "epoch": 0.3101970865467009, |
| "grad_norm": 0.3060204088687897, |
| "kl": 0.0006518363952636719, |
| "learning_rate": 8.245653237555705e-07, |
| "loss": 0.066, |
| "reward": 0.6866211239248514, |
| "reward_std": 1.0839852541685104, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.17629671841859818, |
| "rewards/format_reward": 0.29166667722165585, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1632.5208740234375, |
| "epoch": 0.31191088260497, |
| "grad_norm": 0.3030488193035126, |
| "kl": 0.00047004222869873047, |
| "learning_rate": 8.220696016880687e-07, |
| "loss": 0.0201, |
| "reward": -0.2194829210639, |
| "reward_std": 0.4896611124277115, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.3070765510201454, |
| "rewards/format_reward": 0.11458333767950535, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1587.1458740234375, |
| "epoch": 0.31362467866323906, |
| "grad_norm": 0.2954941987991333, |
| "kl": 0.0003554821014404297, |
| "learning_rate": 8.195606193320136e-07, |
| "loss": 0.0997, |
| "reward": 0.5128022143617272, |
| "reward_std": 0.7717390954494476, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.18813828378915787, |
| "rewards/format_reward": 0.22916666977107525, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1508.0000305175781, |
| "epoch": 0.31533847472150817, |
| "grad_norm": 0.3083325922489166, |
| "kl": 0.0015592575073242188, |
| "learning_rate": 8.170384989716657e-07, |
| "loss": -0.0724, |
| "reward": 0.4367760196328163, |
| "reward_std": 0.8800807446241379, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.19818584620952606, |
| "rewards/format_reward": 0.250000006519258, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1524.9375305175781, |
| "epoch": 0.3170522707797772, |
| "grad_norm": 0.286538302898407, |
| "kl": 0.0006296634674072266, |
| "learning_rate": 8.145033635316128e-07, |
| "loss": 0.0208, |
| "reward": 0.22530196234583855, |
| "reward_std": 0.7582524865865707, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.22987183183431625, |
| "rewards/format_reward": 0.19791667442768812, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1506.3958740234375, |
| "epoch": 0.31876606683804626, |
| "grad_norm": 0.3232147991657257, |
| "kl": 0.00032401084899902344, |
| "learning_rate": 8.119553365707802e-07, |
| "loss": 0.0208, |
| "reward": 0.4753815531730652, |
| "reward_std": 0.7288932427763939, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2013268917798996, |
| "rewards/format_reward": 0.2500000111758709, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1399.0833740234375, |
| "epoch": 0.32047986289631536, |
| "grad_norm": 0.2806786894798279, |
| "kl": 0.0003571510314941406, |
| "learning_rate": 8.093945422764069e-07, |
| "loss": 0.0452, |
| "reward": 1.078583400696516, |
| "reward_std": 1.0053385198116302, |
| "rewards/accuracy_reward": 0.13541667349636555, |
| "rewards/cosine_scaled_reward": -0.1937819980084896, |
| "rewards/format_reward": 0.3854166865348816, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1659.1041870117188, |
| "epoch": 0.3221936589545844, |
| "grad_norm": 0.27923351526260376, |
| "kl": 0.0006756782531738281, |
| "learning_rate": 8.068211054579943e-07, |
| "loss": 0.0204, |
| "reward": 0.07308734953403473, |
| "reward_std": 0.5209638699889183, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.21014533191919327, |
| "rewards/format_reward": 0.14583333861082792, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1439.4166870117188, |
| "epoch": 0.32390745501285345, |
| "grad_norm": 0.337918221950531, |
| "kl": 0.0007171630859375, |
| "learning_rate": 8.04235151541222e-07, |
| "loss": 0.0703, |
| "reward": 0.15663141384720802, |
| "reward_std": 0.5696099773049355, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.19109883531928062, |
| "rewards/format_reward": 0.197916679084301, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1567.0833435058594, |
| "epoch": 0.32562125107112255, |
| "grad_norm": 0.265076220035553, |
| "kl": 0.002460002899169922, |
| "learning_rate": 8.01636806561836e-07, |
| "loss": -0.0344, |
| "reward": 0.11269698292016983, |
| "reward_std": 0.5739325433969498, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.23342406563460827, |
| "rewards/format_reward": 0.2083333395421505, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1663.8541870117188, |
| "epoch": 0.3273350471293916, |
| "grad_norm": 0.2943641245365143, |
| "kl": 0.0004372596740722656, |
| "learning_rate": 7.990261971595048e-07, |
| "loss": 0.0196, |
| "reward": 0.16959162894636393, |
| "reward_std": 0.7194448634982109, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2530036121606827, |
| "rewards/format_reward": 0.1979166753590107, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1417.1458740234375, |
| "epoch": 0.32904884318766064, |
| "grad_norm": 0.3483695685863495, |
| "kl": 0.0011310577392578125, |
| "learning_rate": 7.964034505716476e-07, |
| "loss": -0.0292, |
| "reward": 0.28315007127821445, |
| "reward_std": 0.7390157654881477, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2552552279084921, |
| "rewards/format_reward": 0.2916666716337204, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1268.9375457763672, |
| "epoch": 0.33076263924592975, |
| "grad_norm": 0.329458624124527, |
| "kl": 0.0010833740234375, |
| "learning_rate": 7.93768694627233e-07, |
| "loss": 0.0303, |
| "reward": 0.5155042503029108, |
| "reward_std": 0.8000799417495728, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.2303585298359394, |
| "rewards/format_reward": 0.3750000149011612, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1264.2292175292969, |
| "epoch": 0.3324764353041988, |
| "grad_norm": 0.35155826807022095, |
| "kl": 0.0013065338134765625, |
| "learning_rate": 7.911220577405484e-07, |
| "loss": 0.0121, |
| "reward": 0.6516094729304314, |
| "reward_std": 0.6152506731450558, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.18962617963552475, |
| "rewards/format_reward": 0.3854166753590107, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1593.8125305175781, |
| "epoch": 0.3341902313624679, |
| "grad_norm": 0.30138924717903137, |
| "kl": 0.0005452632904052734, |
| "learning_rate": 7.884636689049422e-07, |
| "loss": -0.0596, |
| "reward": 0.268291431479156, |
| "reward_std": 0.5906689390540123, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.17487580329179764, |
| "rewards/format_reward": 0.19791667070239782, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1491.8958740234375, |
| "epoch": 0.33590402742073694, |
| "grad_norm": 0.28991609811782837, |
| "kl": 0.0017118453979492188, |
| "learning_rate": 7.857936576865356e-07, |
| "loss": -0.0337, |
| "reward": 0.5647330663632601, |
| "reward_std": 0.8028706908226013, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.18598925322294235, |
| "rewards/format_reward": 0.3125000074505806, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1520.1667175292969, |
| "epoch": 0.337617823479006, |
| "grad_norm": 0.30366936326026917, |
| "kl": 0.0009660720825195312, |
| "learning_rate": 7.831121542179086e-07, |
| "loss": -0.0631, |
| "reward": 0.5051862448453903, |
| "reward_std": 0.6083739511668682, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.21957962960004807, |
| "rewards/format_reward": 0.26041666977107525, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1455.9583740234375, |
| "epoch": 0.3393316195372751, |
| "grad_norm": 0.28774720430374146, |
| "kl": 0.0010623931884765625, |
| "learning_rate": 7.804192891917571e-07, |
| "loss": -0.0166, |
| "reward": 0.43268176168203354, |
| "reward_std": 0.5119665786623955, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.23898062482476234, |
| "rewards/format_reward": 0.3333333432674408, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1552.5000305175781, |
| "epoch": 0.34104541559554413, |
| "grad_norm": 0.27825096249580383, |
| "kl": 0.0020427703857421875, |
| "learning_rate": 7.777151938545235e-07, |
| "loss": 0.0261, |
| "reward": 0.457376591861248, |
| "reward_std": 0.711308553814888, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.1980813667178154, |
| "rewards/format_reward": 0.2395833432674408, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1537.6667175292969, |
| "epoch": 0.3427592116538132, |
| "grad_norm": 0.2760595977306366, |
| "kl": 0.0009918212890625, |
| "learning_rate": 7.75e-07, |
| "loss": 0.0019, |
| "reward": 0.6058689560741186, |
| "reward_std": 0.8575867787003517, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.1764862439595163, |
| "rewards/format_reward": 0.3333333507180214, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1451.8958740234375, |
| "epoch": 0.3444730077120823, |
| "grad_norm": 0.29227250814437866, |
| "kl": 0.0012784004211425781, |
| "learning_rate": 7.72273839962904e-07, |
| "loss": -0.0181, |
| "reward": 0.5321098193526268, |
| "reward_std": 0.5402198471128941, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.18361522071063519, |
| "rewards/format_reward": 0.2500000027939677, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1395.0000305175781, |
| "epoch": 0.3461868037703513, |
| "grad_norm": 0.26160165667533875, |
| "kl": 0.0015106201171875, |
| "learning_rate": 7.695368466124296e-07, |
| "loss": 0.0488, |
| "reward": 0.9036880284547806, |
| "reward_std": 0.8476225659251213, |
| "rewards/accuracy_reward": 0.09375000279396772, |
| "rewards/cosine_scaled_reward": -0.18844211474061012, |
| "rewards/format_reward": 0.3854166716337204, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1409.1458740234375, |
| "epoch": 0.34790059982862037, |
| "grad_norm": 0.3121320605278015, |
| "kl": 0.0012507438659667969, |
| "learning_rate": 7.667891533457718e-07, |
| "loss": 0.0627, |
| "reward": 1.0111969839781523, |
| "reward_std": 1.1430891081690788, |
| "rewards/accuracy_reward": 0.11458333488553762, |
| "rewards/cosine_scaled_reward": -0.21238414198160172, |
| "rewards/format_reward": 0.3854166753590107, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1446.9167175292969, |
| "epoch": 0.3496143958868895, |
| "grad_norm": 0.27098214626312256, |
| "kl": 0.0015354156494140625, |
| "learning_rate": 7.640308940816239e-07, |
| "loss": 0.0206, |
| "reward": 0.42572443559765816, |
| "reward_std": 0.3084217198193073, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.18036598339676857, |
| "rewards/format_reward": 0.2187500037252903, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1411.2708740234375, |
| "epoch": 0.3513281919451585, |
| "grad_norm": 0.28694698214530945, |
| "kl": 0.0013275146484375, |
| "learning_rate": 7.612622032536507e-07, |
| "loss": 0.1296, |
| "reward": 0.1224188543856144, |
| "reward_std": 0.6760578900575638, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.3061027526855469, |
| "rewards/format_reward": 0.2604166744276881, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1423.5209045410156, |
| "epoch": 0.35304198800342756, |
| "grad_norm": 0.355116605758667, |
| "kl": 0.0011377334594726562, |
| "learning_rate": 7.584832158039378e-07, |
| "loss": 0.0178, |
| "reward": 0.30912495171651244, |
| "reward_std": 0.7893542125821114, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.20777561515569687, |
| "rewards/format_reward": 0.2500000102445483, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1428.8125305175781, |
| "epoch": 0.35475578406169667, |
| "grad_norm": 0.2700238823890686, |
| "kl": 0.0029344558715820312, |
| "learning_rate": 7.556940671764124e-07, |
| "loss": 0.0719, |
| "reward": 0.4809890575706959, |
| "reward_std": 0.7781959772109985, |
| "rewards/accuracy_reward": 0.06250000093132257, |
| "rewards/cosine_scaled_reward": -0.22299901768565178, |
| "rewards/format_reward": 0.2395833395421505, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1418.7500305175781, |
| "epoch": 0.3564695801199657, |
| "grad_norm": 0.3213181495666504, |
| "kl": 0.0018987655639648438, |
| "learning_rate": 7.528948933102438e-07, |
| "loss": -0.1139, |
| "reward": 0.30493446439504623, |
| "reward_std": 0.75405253469944, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.22520049661397934, |
| "rewards/format_reward": 0.22916667442768812, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1472.1458740234375, |
| "epoch": 0.3581833761782348, |
| "grad_norm": 0.295888215303421, |
| "kl": 0.001293182373046875, |
| "learning_rate": 7.500858306332172e-07, |
| "loss": 0.0205, |
| "reward": 0.7049083486199379, |
| "reward_std": 0.7757160514593124, |
| "rewards/accuracy_reward": 0.07291666977107525, |
| "rewards/cosine_scaled_reward": -0.14897773665143177, |
| "rewards/format_reward": 0.26041667722165585, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1384.4583740234375, |
| "epoch": 0.35989717223650386, |
| "grad_norm": 0.3134666681289673, |
| "kl": 0.0015621185302734375, |
| "learning_rate": 7.472670160550848e-07, |
| "loss": 0.0936, |
| "reward": 0.6157244890928268, |
| "reward_std": 0.9224071279168129, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.1860289741307497, |
| "rewards/format_reward": 0.3750000149011612, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1649.0625610351562, |
| "epoch": 0.3616109682947729, |
| "grad_norm": 0.23310723900794983, |
| "kl": 0.0006375312805175781, |
| "learning_rate": 7.444385869608921e-07, |
| "loss": 0.0379, |
| "reward": -0.043950184248387814, |
| "reward_std": 0.3435293883085251, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2602936290204525, |
| "rewards/format_reward": 0.19791666977107525, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1412.1667175292969, |
| "epoch": 0.363324764353042, |
| "grad_norm": 0.27200737595558167, |
| "kl": 0.0014286041259765625, |
| "learning_rate": 7.416006812042827e-07, |
| "loss": 0.0497, |
| "reward": 0.9091778621077538, |
| "reward_std": 0.7571589723229408, |
| "rewards/accuracy_reward": 0.09375000186264515, |
| "rewards/cosine_scaled_reward": -0.11946619441732764, |
| "rewards/format_reward": 0.2708333386108279, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1543.1458435058594, |
| "epoch": 0.36503856041131105, |
| "grad_norm": 0.24152739346027374, |
| "kl": 0.0015354156494140625, |
| "learning_rate": 7.387534371007797e-07, |
| "loss": -0.0126, |
| "reward": 0.011342905461788177, |
| "reward_std": 0.37173977866768837, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2708110474050045, |
| "rewards/format_reward": 0.25000000186264515, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1498.9583740234375, |
| "epoch": 0.3667523564695801, |
| "grad_norm": 0.29263201355934143, |
| "kl": 0.001026153564453125, |
| "learning_rate": 7.358969934210438e-07, |
| "loss": 0.0561, |
| "reward": 0.2628788137808442, |
| "reward_std": 0.3592139929533005, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2303277626633644, |
| "rewards/format_reward": 0.19791666977107525, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1469.6250305175781, |
| "epoch": 0.3684661525278492, |
| "grad_norm": 0.24959833920001984, |
| "kl": 0.0009860992431640625, |
| "learning_rate": 7.330314893841101e-07, |
| "loss": 0.0211, |
| "reward": 0.5021949373185635, |
| "reward_std": 0.937240332365036, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.21530525013804436, |
| "rewards/format_reward": 0.3125000111758709, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1592.7708435058594, |
| "epoch": 0.37017994858611825, |
| "grad_norm": 0.281783789396286, |
| "kl": 0.0015850067138671875, |
| "learning_rate": 7.301570646506027e-07, |
| "loss": -0.0691, |
| "reward": 0.4340474624186754, |
| "reward_std": 0.7465260550379753, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.2362687811255455, |
| "rewards/format_reward": 0.19791667349636555, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1485.6250305175781, |
| "epoch": 0.3718937446443873, |
| "grad_norm": 0.2756541967391968, |
| "kl": 0.0015964508056640625, |
| "learning_rate": 7.27273859315928e-07, |
| "loss": -0.113, |
| "reward": 0.5143733620643616, |
| "reward_std": 0.8895893320441246, |
| "rewards/accuracy_reward": 0.06250000093132257, |
| "rewards/cosine_scaled_reward": -0.1793924793601036, |
| "rewards/format_reward": 0.21875000279396772, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1517.3750305175781, |
| "epoch": 0.3736075407026564, |
| "grad_norm": 0.29070937633514404, |
| "kl": 0.0009374618530273438, |
| "learning_rate": 7.243820139034464e-07, |
| "loss": -0.0237, |
| "reward": 0.8206158205866814, |
| "reward_std": 1.0968997925519943, |
| "rewards/accuracy_reward": 0.07291666977107525, |
| "rewards/cosine_scaled_reward": -0.12784741912037134, |
| "rewards/format_reward": 0.27083334140479565, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1627.6458740234375, |
| "epoch": 0.37532133676092544, |
| "grad_norm": 0.3091233968734741, |
| "kl": 0.0029497146606445312, |
| "learning_rate": 7.214816693576234e-07, |
| "loss": -0.0081, |
| "reward": 0.5958885625004768, |
| "reward_std": 0.9548313468694687, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.165425858926028, |
| "rewards/format_reward": 0.23958333861082792, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1657.9583740234375, |
| "epoch": 0.37703513281919454, |
| "grad_norm": 0.30196475982666016, |
| "kl": 0.0020732879638671875, |
| "learning_rate": 7.185729670371604e-07, |
| "loss": -0.0144, |
| "reward": -0.12533910013735294, |
| "reward_std": 0.3947201892733574, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2612520717084408, |
| "rewards/format_reward": 0.14583333767950535, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1646.6458740234375, |
| "epoch": 0.3787489288774636, |
| "grad_norm": 0.26658761501312256, |
| "kl": 0.0012149810791015625, |
| "learning_rate": 7.156560487081051e-07, |
| "loss": -0.0114, |
| "reward": 0.18843666091561317, |
| "reward_std": 0.7580625228583813, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2518426850438118, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1469.4791870117188, |
| "epoch": 0.38046272493573263, |
| "grad_norm": 0.29353833198547363, |
| "kl": 0.0029439926147460938, |
| "learning_rate": 7.127310565369415e-07, |
| "loss": 0.0103, |
| "reward": 0.5816408507525921, |
| "reward_std": 0.511856883764267, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.19628694094717503, |
| "rewards/format_reward": 0.291666672565043, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1677.5417175292969, |
| "epoch": 0.38217652099400173, |
| "grad_norm": 0.24983949959278107, |
| "kl": 0.00212860107421875, |
| "learning_rate": 7.097981330836616e-07, |
| "loss": -0.0388, |
| "reward": 0.10370806977152824, |
| "reward_std": 0.7805009186267853, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2444031797349453, |
| "rewards/format_reward": 0.12500000186264515, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1588.0417175292969, |
| "epoch": 0.3838903170522708, |
| "grad_norm": 0.3074623942375183, |
| "kl": 0.0014591217041015625, |
| "learning_rate": 7.068574212948169e-07, |
| "loss": -0.0364, |
| "reward": -0.21326511725783348, |
| "reward_std": 0.40029212832450867, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.29444269090890884, |
| "rewards/format_reward": 0.10416667070239782, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1470.4166870117188, |
| "epoch": 0.3856041131105398, |
| "grad_norm": 0.2668305039405823, |
| "kl": 0.004240512847900391, |
| "learning_rate": 7.039090644965509e-07, |
| "loss": -0.0444, |
| "reward": 0.43112599570304155, |
| "reward_std": 1.025677740573883, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.2224056925624609, |
| "rewards/format_reward": 0.21875000651925802, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1524.3542175292969, |
| "epoch": 0.3873179091688089, |
| "grad_norm": 0.24207906424999237, |
| "kl": 0.001285552978515625, |
| "learning_rate": 7.009532063876148e-07, |
| "loss": -0.0133, |
| "reward": 0.20098010450601578, |
| "reward_std": 0.4968443959951401, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.21850886940956116, |
| "rewards/format_reward": 0.2083333395421505, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1687.1250305175781, |
| "epoch": 0.389031705227078, |
| "grad_norm": 0.2755184769630432, |
| "kl": 0.0006885528564453125, |
| "learning_rate": 6.979899910323624e-07, |
| "loss": 0.0313, |
| "reward": 0.1163886021822691, |
| "reward_std": 0.5962348952889442, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.23180323839187622, |
| "rewards/format_reward": 0.16666666977107525, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1521.3541870117188, |
| "epoch": 0.390745501285347, |
| "grad_norm": 0.2736872136592865, |
| "kl": 0.00174713134765625, |
| "learning_rate": 6.950195628537299e-07, |
| "loss": 0.0192, |
| "reward": 0.6056804731488228, |
| "reward_std": 0.9653726145625114, |
| "rewards/accuracy_reward": 0.07291666977107525, |
| "rewards/cosine_scaled_reward": -0.16172956302762032, |
| "rewards/format_reward": 0.2812500037252903, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1430.7709045410156, |
| "epoch": 0.3924592973436161, |
| "grad_norm": 0.2898569703102112, |
| "kl": 0.003772735595703125, |
| "learning_rate": 6.920420666261961e-07, |
| "loss": -0.1013, |
| "reward": 0.16834751144051552, |
| "reward_std": 0.8317932933568954, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.259422168135643, |
| "rewards/format_reward": 0.2187500037252903, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1527.5625, |
| "epoch": 0.39417309340188517, |
| "grad_norm": 0.2458394318819046, |
| "kl": 0.00273895263671875, |
| "learning_rate": 6.890576474687263e-07, |
| "loss": 0.0225, |
| "reward": 0.36850229650735855, |
| "reward_std": 0.44148915261030197, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.22217821329832077, |
| "rewards/format_reward": 0.27083334419876337, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1635.2292175292969, |
| "epoch": 0.39588688946015427, |
| "grad_norm": 0.24855723977088928, |
| "kl": 0.001384735107421875, |
| "learning_rate": 6.860664508377001e-07, |
| "loss": -0.0123, |
| "reward": 0.3638776633888483, |
| "reward_std": 0.9782525599002838, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.19856508448719978, |
| "rewards/format_reward": 0.21875000558793545, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1465.6875, |
| "epoch": 0.3976006855184233, |
| "grad_norm": 0.2706748843193054, |
| "kl": 0.0015106201171875, |
| "learning_rate": 6.83068622519821e-07, |
| "loss": -0.0079, |
| "reward": 0.6824747771024704, |
| "reward_std": 1.202951930463314, |
| "rewards/accuracy_reward": 0.08333333674818277, |
| "rewards/cosine_scaled_reward": -0.1890408918261528, |
| "rewards/format_reward": 0.2708333386108279, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1183.6458587646484, |
| "epoch": 0.39931448157669236, |
| "grad_norm": 0.3087099492549896, |
| "kl": 0.0015783309936523438, |
| "learning_rate": 6.800643086250121e-07, |
| "loss": -0.0562, |
| "reward": 1.6357879862189293, |
| "reward_std": 0.9204326570034027, |
| "rewards/accuracy_reward": 0.1562500037252903, |
| "rewards/cosine_scaled_reward": -0.08350442518712953, |
| "rewards/format_reward": 0.5625000223517418, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1545.1458740234375, |
| "epoch": 0.40102827763496146, |
| "grad_norm": 0.30824851989746094, |
| "kl": 0.0018053054809570312, |
| "learning_rate": 6.770536555792944e-07, |
| "loss": -0.0356, |
| "reward": 0.3720626160502434, |
| "reward_std": 0.5978400260210037, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.2530326321721077, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1724.4166870117188, |
| "epoch": 0.4027420736932305, |
| "grad_norm": 0.21939213573932648, |
| "kl": 0.0011310577392578125, |
| "learning_rate": 6.740368101176495e-07, |
| "loss": -0.0541, |
| "reward": -0.09954050742089748, |
| "reward_std": 0.5700792819261551, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.25520598143339157, |
| "rewards/format_reward": 0.07291666697710752, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1656.7708740234375, |
| "epoch": 0.40445586975149955, |
| "grad_norm": 0.24315086007118225, |
| "kl": 0.0017242431640625, |
| "learning_rate": 6.710139192768694e-07, |
| "loss": 0.0181, |
| "reward": -0.08899100869894028, |
| "reward_std": 0.567382175475359, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2840782068669796, |
| "rewards/format_reward": 0.0937500037252903, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1391.2917175292969, |
| "epoch": 0.40616966580976865, |
| "grad_norm": 0.284648060798645, |
| "kl": 0.004982471466064453, |
| "learning_rate": 6.679851303883891e-07, |
| "loss": -0.0752, |
| "reward": 0.4182049632072449, |
| "reward_std": 0.6335307955741882, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.21466262266039848, |
| "rewards/format_reward": 0.31250000558793545, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1576.8542175292969, |
| "epoch": 0.4078834618680377, |
| "grad_norm": 0.37806493043899536, |
| "kl": 0.0016613006591796875, |
| "learning_rate": 6.649505910711058e-07, |
| "loss": -0.0262, |
| "reward": 0.23228356195613742, |
| "reward_std": 0.5114509798586369, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.20781026035547256, |
| "rewards/format_reward": 0.14583333674818277, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1438.1250305175781, |
| "epoch": 0.40959725792630675, |
| "grad_norm": 0.2724723517894745, |
| "kl": 0.00337982177734375, |
| "learning_rate": 6.619104492241847e-07, |
| "loss": -0.0632, |
| "reward": 0.2005984801799059, |
| "reward_std": 0.46372513473033905, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2599928639829159, |
| "rewards/format_reward": 0.28125000838190317, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1323.2708587646484, |
| "epoch": 0.41131105398457585, |
| "grad_norm": 0.29104921221733093, |
| "kl": 0.0015172958374023438, |
| "learning_rate": 6.588648530198504e-07, |
| "loss": 0.0754, |
| "reward": 1.0371160432696342, |
| "reward_std": 1.0028871893882751, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.16367125883698463, |
| "rewards/format_reward": 0.4166666753590107, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1315.8541870117188, |
| "epoch": 0.4130248500428449, |
| "grad_norm": 0.3675242066383362, |
| "kl": 0.00168609619140625, |
| "learning_rate": 6.558139508961654e-07, |
| "loss": -0.0468, |
| "reward": 0.6890245275571942, |
| "reward_std": 0.829789325594902, |
| "rewards/accuracy_reward": 0.07291666977107525, |
| "rewards/cosine_scaled_reward": -0.20996215008199215, |
| "rewards/format_reward": 0.385416679084301, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1711.6250305175781, |
| "epoch": 0.414738646101114, |
| "grad_norm": 0.27449020743370056, |
| "kl": 0.0006427764892578125, |
| "learning_rate": 6.527578915497951e-07, |
| "loss": -0.0047, |
| "reward": 0.0678191315382719, |
| "reward_std": 0.6721196174621582, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2533269338309765, |
| "rewards/format_reward": 0.1666666679084301, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1564.2292175292969, |
| "epoch": 0.41645244215938304, |
| "grad_norm": 0.2841816842556, |
| "kl": 0.001232147216796875, |
| "learning_rate": 6.496968239287603e-07, |
| "loss": 0.0798, |
| "reward": -0.010489307343959808, |
| "reward_std": 0.6119135469198227, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.31749165803194046, |
| "rewards/format_reward": 0.21875000931322575, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1542.7083740234375, |
| "epoch": 0.4181662382176521, |
| "grad_norm": 0.2418055236339569, |
| "kl": 0.0013904571533203125, |
| "learning_rate": 6.466308972251785e-07, |
| "loss": 0.0006, |
| "reward": 0.42967063235118985, |
| "reward_std": 0.5240038335323334, |
| "rewards/accuracy_reward": 0.0520833358168602, |
| "rewards/cosine_scaled_reward": -0.22576111694797873, |
| "rewards/format_reward": 0.22916667256504297, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1163.6875305175781, |
| "epoch": 0.4198800342759212, |
| "grad_norm": 0.3541621267795563, |
| "kl": 0.0019626617431640625, |
| "learning_rate": 6.435602608679916e-07, |
| "loss": -0.0503, |
| "reward": 1.0524671040475368, |
| "reward_std": 0.8917372450232506, |
| "rewards/accuracy_reward": 0.07291666977107525, |
| "rewards/cosine_scaled_reward": -0.17973799630999565, |
| "rewards/format_reward": 0.6145833432674408, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1451.6250610351562, |
| "epoch": 0.42159383033419023, |
| "grad_norm": 0.30440303683280945, |
| "kl": 0.0012979507446289062, |
| "learning_rate": 6.404850645156841e-07, |
| "loss": -0.0257, |
| "reward": 0.6202996233478189, |
| "reward_std": 0.7275250256061554, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.16593201458454132, |
| "rewards/format_reward": 0.31250001676380634, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1685.4583435058594, |
| "epoch": 0.4233076263924593, |
| "grad_norm": 0.28518855571746826, |
| "kl": 0.0011157989501953125, |
| "learning_rate": 6.374054580489873e-07, |
| "loss": -0.025, |
| "reward": -0.02131518442183733, |
| "reward_std": 0.5307684168219566, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.23101667314767838, |
| "rewards/format_reward": 0.11458333488553762, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1473.8125305175781, |
| "epoch": 0.4250214224507284, |
| "grad_norm": 0.23807476460933685, |
| "kl": 0.0013265609741210938, |
| "learning_rate": 6.343215915635761e-07, |
| "loss": -0.0614, |
| "reward": 0.7047823406755924, |
| "reward_std": 0.7143717184662819, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.1652532685548067, |
| "rewards/format_reward": 0.2812500027939677, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1546.3541870117188, |
| "epoch": 0.4267352185089974, |
| "grad_norm": 0.24748165905475616, |
| "kl": 0.0014638900756835938, |
| "learning_rate": 6.31233615362752e-07, |
| "loss": 0.0185, |
| "reward": 0.6113807391375303, |
| "reward_std": 0.5387562289834023, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.19036031886935234, |
| "rewards/format_reward": 0.2187500037252903, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1507.4375305175781, |
| "epoch": 0.4284490145672665, |
| "grad_norm": 0.2894173860549927, |
| "kl": 0.0014209747314453125, |
| "learning_rate": 6.281416799501187e-07, |
| "loss": -0.0473, |
| "reward": 0.19241389445960522, |
| "reward_std": 0.7252123057842255, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2510671392083168, |
| "rewards/format_reward": 0.25000000186264515, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1443.2083740234375, |
| "epoch": 0.4301628106255356, |
| "grad_norm": 0.4095551669597626, |
| "kl": 0.0017261505126953125, |
| "learning_rate": 6.25045936022246e-07, |
| "loss": -0.1353, |
| "reward": 0.6435118420049548, |
| "reward_std": 0.934768907725811, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.1795162484049797, |
| "rewards/format_reward": 0.3020833358168602, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1465.9375610351562, |
| "epoch": 0.4318766066838046, |
| "grad_norm": 0.275666743516922, |
| "kl": 0.0014324188232421875, |
| "learning_rate": 6.219465344613258e-07, |
| "loss": -0.018, |
| "reward": 0.12436750531196594, |
| "reward_std": 0.7275985479354858, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.26183638721704483, |
| "rewards/format_reward": 0.18750000279396772, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1510.0417175292969, |
| "epoch": 0.43359040274207367, |
| "grad_norm": 0.25496789813041687, |
| "kl": 0.0009136199951171875, |
| "learning_rate": 6.188436263278172e-07, |
| "loss": -0.016, |
| "reward": 0.5188035815954208, |
| "reward_std": 0.9701652638614178, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.2066810093820095, |
| "rewards/format_reward": 0.1770833395421505, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1549.9792175292969, |
| "epoch": 0.43530419880034277, |
| "grad_norm": 0.27471932768821716, |
| "kl": 0.0010223388671875, |
| "learning_rate": 6.157373628530852e-07, |
| "loss": -0.0365, |
| "reward": 0.5654481025412679, |
| "reward_std": 1.0282247960567474, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.1806558482348919, |
| "rewards/format_reward": 0.2812500037252903, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1348.2083740234375, |
| "epoch": 0.4370179948586118, |
| "grad_norm": 0.30511415004730225, |
| "kl": 0.0012774467468261719, |
| "learning_rate": 6.126278954320294e-07, |
| "loss": 0.0483, |
| "reward": 0.8795673847198486, |
| "reward_std": 0.8337312787771225, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.1382422624155879, |
| "rewards/format_reward": 0.3333333432674408, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1538.2083740234375, |
| "epoch": 0.4387317909168809, |
| "grad_norm": 0.2755931615829468, |
| "kl": 0.003383636474609375, |
| "learning_rate": 6.095153756157051e-07, |
| "loss": 0.0742, |
| "reward": 0.2700856775045395, |
| "reward_std": 0.3797752112150192, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.25267578288912773, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1461.8959045410156, |
| "epoch": 0.44044558697514996, |
| "grad_norm": 0.3680107891559601, |
| "kl": 0.002166748046875, |
| "learning_rate": 6.06399955103937e-07, |
| "loss": 0.1579, |
| "reward": 0.39618358900770545, |
| "reward_std": 0.7873297780752182, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.23296307399868965, |
| "rewards/format_reward": 0.2395833395421505, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1543.0208740234375, |
| "epoch": 0.442159383033419, |
| "grad_norm": 0.27821409702301025, |
| "kl": 0.0012140274047851562, |
| "learning_rate": 6.032817857379256e-07, |
| "loss": 0.0219, |
| "reward": 0.36168060451745987, |
| "reward_std": 0.5424905493855476, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.24902323260903358, |
| "rewards/format_reward": 0.22916667070239782, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1361.4375305175781, |
| "epoch": 0.4438731790916881, |
| "grad_norm": 0.3345107436180115, |
| "kl": 0.0014476776123046875, |
| "learning_rate": 6.001610194928464e-07, |
| "loss": 0.0802, |
| "reward": 1.179038206115365, |
| "reward_std": 1.1666965633630753, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.1345681119710207, |
| "rewards/format_reward": 0.43750001303851604, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1471.3958740234375, |
| "epoch": 0.44558697514995715, |
| "grad_norm": 0.283227801322937, |
| "kl": 0.0009584426879882812, |
| "learning_rate": 5.97037808470444e-07, |
| "loss": 0.0181, |
| "reward": 0.19981666887179017, |
| "reward_std": 0.7603968679904938, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.28081241995096207, |
| "rewards/format_reward": 0.2708333386108279, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1480.2292175292969, |
| "epoch": 0.4473007712082262, |
| "grad_norm": 0.2809438705444336, |
| "kl": 0.00098419189453125, |
| "learning_rate": 5.939123048916173e-07, |
| "loss": -0.0626, |
| "reward": 0.6192706078290939, |
| "reward_std": 0.9330656826496124, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.22592685744166374, |
| "rewards/format_reward": 0.3750000074505806, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1499.7917175292969, |
| "epoch": 0.4490145672664953, |
| "grad_norm": 0.2977607846260071, |
| "kl": 0.0025501251220703125, |
| "learning_rate": 5.907846610890011e-07, |
| "loss": 0.0765, |
| "reward": 0.40138836577534676, |
| "reward_std": 1.058370865881443, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.18452044390141964, |
| "rewards/format_reward": 0.21875000558793545, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1516.2291870117188, |
| "epoch": 0.45072836332476435, |
| "grad_norm": 0.2541120648384094, |
| "kl": 0.0012226104736328125, |
| "learning_rate": 5.87655029499542e-07, |
| "loss": 0.0006, |
| "reward": 0.20188701339066029, |
| "reward_std": 0.5907558128237724, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.215114988386631, |
| "rewards/format_reward": 0.2395833395421505, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1588.2083435058594, |
| "epoch": 0.4524421593830334, |
| "grad_norm": 0.369785338640213, |
| "kl": 0.003452301025390625, |
| "learning_rate": 5.845235626570683e-07, |
| "loss": 0.1067, |
| "reward": -0.17164241895079613, |
| "reward_std": 0.37897656112909317, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2791437990963459, |
| "rewards/format_reward": 0.15625000651925802, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1688.2083740234375, |
| "epoch": 0.4541559554413025, |
| "grad_norm": 0.25646331906318665, |
| "kl": 0.0008211135864257812, |
| "learning_rate": 5.813904131848564e-07, |
| "loss": 0.0496, |
| "reward": 0.40768040530383587, |
| "reward_std": 0.47176536452025175, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.11216204985976219, |
| "rewards/format_reward": 0.0833333358168602, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1518.7916870117188, |
| "epoch": 0.45586975149957154, |
| "grad_norm": 0.2357741892337799, |
| "kl": 0.000965118408203125, |
| "learning_rate": 5.78255733788191e-07, |
| "loss": -0.0472, |
| "reward": 0.5058878529816866, |
| "reward_std": 0.9048085063695908, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.1941013392060995, |
| "rewards/format_reward": 0.2291666753590107, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1370.0208740234375, |
| "epoch": 0.45758354755784064, |
| "grad_norm": 0.2968303859233856, |
| "kl": 0.00202178955078125, |
| "learning_rate": 5.751196772469237e-07, |
| "loss": 0.0396, |
| "reward": 0.4842541292309761, |
| "reward_std": 0.7571203708648682, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2155583193525672, |
| "rewards/format_reward": 0.3333333358168602, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1385.8333740234375, |
| "epoch": 0.4592973436161097, |
| "grad_norm": 0.2968428432941437, |
| "kl": 0.0023469924926757812, |
| "learning_rate": 5.71982396408026e-07, |
| "loss": -0.017, |
| "reward": 0.6936724968254566, |
| "reward_std": 0.8312683999538422, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.22146771964617074, |
| "rewards/format_reward": 0.3645833469927311, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1570.6667175292969, |
| "epoch": 0.46101113967437873, |
| "grad_norm": 0.2661321461200714, |
| "kl": 0.0008296966552734375, |
| "learning_rate": 5.688440441781398e-07, |
| "loss": 0.0054, |
| "reward": 0.13332478515803814, |
| "reward_std": 0.4149567186832428, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.23616338148713112, |
| "rewards/format_reward": 0.2187500111758709, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1490.6875610351562, |
| "epoch": 0.46272493573264784, |
| "grad_norm": 0.2545984089374542, |
| "kl": 0.00135040283203125, |
| "learning_rate": 5.657047735161255e-07, |
| "loss": 0.0919, |
| "reward": 0.7821074835956097, |
| "reward_std": 0.6425863802433014, |
| "rewards/accuracy_reward": 0.09375000186264515, |
| "rewards/cosine_scaled_reward": -0.1767166256904602, |
| "rewards/format_reward": 0.3125000111758709, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1521.8750610351562, |
| "epoch": 0.4644387317909169, |
| "grad_norm": 0.2771352231502533, |
| "kl": 0.0013113021850585938, |
| "learning_rate": 5.625647374256061e-07, |
| "loss": -0.0375, |
| "reward": 0.3960751476697624, |
| "reward_std": 0.46416520327329636, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2203481704927981, |
| "rewards/format_reward": 0.2500000111758709, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1412.1042175292969, |
| "epoch": 0.4661525278491859, |
| "grad_norm": 0.27568578720092773, |
| "kl": 0.00176239013671875, |
| "learning_rate": 5.594240889475106e-07, |
| "loss": 0.0309, |
| "reward": 0.44331623800098896, |
| "reward_std": 0.6773833259940147, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.21556473523378372, |
| "rewards/format_reward": 0.3020833432674408, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1356.6041870117188, |
| "epoch": 0.46786632390745503, |
| "grad_norm": 0.27975478768348694, |
| "kl": 0.0014009475708007812, |
| "learning_rate": 5.562829811526154e-07, |
| "loss": -0.0274, |
| "reward": 0.7240446954965591, |
| "reward_std": 1.045374572277069, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/cosine_scaled_reward": -0.21347414329648018, |
| "rewards/format_reward": 0.354166679084301, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1636.6667175292969, |
| "epoch": 0.4695801199657241, |
| "grad_norm": 0.2683631479740143, |
| "kl": 0.0010700225830078125, |
| "learning_rate": 5.531415671340826e-07, |
| "loss": -0.0025, |
| "reward": 0.17355282232165337, |
| "reward_std": 0.3705536052584648, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2301059477031231, |
| "rewards/format_reward": 0.2812500074505806, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1553.2083740234375, |
| "epoch": 0.4712939160239931, |
| "grad_norm": 0.33635562658309937, |
| "kl": 0.0011768341064453125, |
| "learning_rate": 5.5e-07, |
| "loss": 0.0588, |
| "reward": 0.36671384796500206, |
| "reward_std": 0.39920416846871376, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.19434788078069687, |
| "rewards/format_reward": 0.21875, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1501.6042175292969, |
| "epoch": 0.4730077120822622, |
| "grad_norm": 0.288104385137558, |
| "kl": 0.0024690628051757812, |
| "learning_rate": 5.468584328659172e-07, |
| "loss": -0.0367, |
| "reward": 0.35132842278108, |
| "reward_std": 0.8972981795668602, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2235843949019909, |
| "rewards/format_reward": 0.2500000111758709, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1590.3333740234375, |
| "epoch": 0.47472150814053127, |
| "grad_norm": 0.24080416560173035, |
| "kl": 0.0013065338134765625, |
| "learning_rate": 5.437170188473847e-07, |
| "loss": 0.0462, |
| "reward": -0.03010409977287054, |
| "reward_std": 0.49339213222265244, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.24666466563940048, |
| "rewards/format_reward": 0.15625000186264515, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1475.8333435058594, |
| "epoch": 0.47643530419880037, |
| "grad_norm": 0.25690314173698425, |
| "kl": 0.0014476776123046875, |
| "learning_rate": 5.405759110524894e-07, |
| "loss": 0.0637, |
| "reward": 0.5569645212963223, |
| "reward_std": 0.9936123788356781, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.23540310934185982, |
| "rewards/format_reward": 0.3437500074505806, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1678.9791870117188, |
| "epoch": 0.4781491002570694, |
| "grad_norm": 0.24415026605129242, |
| "kl": 0.00095367431640625, |
| "learning_rate": 5.37435262574394e-07, |
| "loss": 0.0144, |
| "reward": 0.16477025300264359, |
| "reward_std": 0.6051457226276398, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.20591148734092712, |
| "rewards/format_reward": 0.1979166716337204, |
| "step": 279 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1506.6250305175781, |
| "epoch": 0.47986289631533846, |
| "grad_norm": 0.26392680406570435, |
| "kl": 0.0023336410522460938, |
| "learning_rate": 5.342952264838747e-07, |
| "loss": 0.0013, |
| "reward": 0.4733018707484007, |
| "reward_std": 0.7523324713110924, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.2140564490109682, |
| "rewards/format_reward": 0.25000000931322575, |
| "step": 280 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1287.0000305175781, |
| "epoch": 0.48157669237360756, |
| "grad_norm": 0.29941684007644653, |
| "kl": 0.0029144287109375, |
| "learning_rate": 5.311559558218603e-07, |
| "loss": -0.0345, |
| "reward": 0.7630709037184715, |
| "reward_std": 0.6651666909456253, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.15811054781079292, |
| "rewards/format_reward": 0.3229166716337204, |
| "step": 281 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1531.1667175292969, |
| "epoch": 0.4832904884318766, |
| "grad_norm": 0.2747533917427063, |
| "kl": 0.0010986328125, |
| "learning_rate": 5.28017603591974e-07, |
| "loss": 0.0652, |
| "reward": 0.2705502174794674, |
| "reward_std": 0.5016405843198299, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.2651446685194969, |
| "rewards/format_reward": 0.23958334419876337, |
| "step": 282 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1673.5625305175781, |
| "epoch": 0.48500428449014565, |
| "grad_norm": 0.22642135620117188, |
| "kl": 0.0008697509765625, |
| "learning_rate": 5.248803227530763e-07, |
| "loss": -0.0168, |
| "reward": 0.13469044864177704, |
| "reward_std": 0.5786846578121185, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.21285491064190865, |
| "rewards/format_reward": 0.17708333488553762, |
| "step": 283 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1504.7916870117188, |
| "epoch": 0.48671808054841476, |
| "grad_norm": 0.2727656662464142, |
| "kl": 0.0019969940185546875, |
| "learning_rate": 5.21744266211809e-07, |
| "loss": 0.0987, |
| "reward": 0.17584533244371414, |
| "reward_std": 0.9530358016490936, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.26050129532814026, |
| "rewards/format_reward": 0.13541666977107525, |
| "step": 284 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1625.1042175292969, |
| "epoch": 0.4884318766066838, |
| "grad_norm": 0.2627033293247223, |
| "kl": 0.00135040283203125, |
| "learning_rate": 5.186095868151436e-07, |
| "loss": -0.0045, |
| "reward": 0.33440714702010155, |
| "reward_std": 0.4926134943962097, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.23611152917146683, |
| "rewards/format_reward": 0.2291666716337204, |
| "step": 285 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1458.3333435058594, |
| "epoch": 0.49014567266495285, |
| "grad_norm": 0.2648809850215912, |
| "kl": 0.0019407272338867188, |
| "learning_rate": 5.154764373429315e-07, |
| "loss": -0.0038, |
| "reward": 0.43773437407799065, |
| "reward_std": 0.8458793424069881, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/cosine_scaled_reward": -0.17636944695550483, |
| "rewards/format_reward": 0.2291666679084301, |
| "step": 286 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1603.5416870117188, |
| "epoch": 0.49185946872322195, |
| "grad_norm": 0.2246415913105011, |
| "kl": 0.0008897781372070312, |
| "learning_rate": 5.123449705004581e-07, |
| "loss": -0.0038, |
| "reward": 0.3730207160115242, |
| "reward_std": 0.6528768911957741, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.20618611853569746, |
| "rewards/format_reward": 0.19791666977107525, |
| "step": 287 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1618.7917175292969, |
| "epoch": 0.493573264781491, |
| "grad_norm": 0.38273441791534424, |
| "kl": 0.0009946823120117188, |
| "learning_rate": 5.09215338910999e-07, |
| "loss": -0.0405, |
| "reward": 0.22999665327370167, |
| "reward_std": 0.5767201408743858, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2371416799724102, |
| "rewards/format_reward": 0.260416679084301, |
| "step": 288 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1577.8125305175781, |
| "epoch": 0.4952870608397601, |
| "grad_norm": 0.25025105476379395, |
| "kl": 0.0012979507446289062, |
| "learning_rate": 5.060876951083828e-07, |
| "loss": -0.0036, |
| "reward": 0.15548935439437628, |
| "reward_std": 0.5783236026763916, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.20484132319688797, |
| "rewards/format_reward": 0.17708333488553762, |
| "step": 289 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1389.0834045410156, |
| "epoch": 0.49700085689802914, |
| "grad_norm": 0.35921505093574524, |
| "kl": 0.0019588470458984375, |
| "learning_rate": 5.02962191529556e-07, |
| "loss": -0.0026, |
| "reward": 0.5405751286889426, |
| "reward_std": 0.9654501527547836, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.19835910387337208, |
| "rewards/format_reward": 0.30208334140479565, |
| "step": 290 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1685.0, |
| "epoch": 0.4987146529562982, |
| "grad_norm": 0.26681071519851685, |
| "kl": 0.0010046958923339844, |
| "learning_rate": 4.998389805071536e-07, |
| "loss": 0.0389, |
| "reward": -0.14831870794296265, |
| "reward_std": 0.35369744896888733, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.26506757736206055, |
| "rewards/format_reward": 0.10416666977107525, |
| "step": 291 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1354.5000305175781, |
| "epoch": 0.5004284490145673, |
| "grad_norm": 0.326076477766037, |
| "kl": 0.00252532958984375, |
| "learning_rate": 4.967182142620745e-07, |
| "loss": 0.0549, |
| "reward": 0.5435319095849991, |
| "reward_std": 0.6741289496421814, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.22746334969997406, |
| "rewards/format_reward": 0.3020833432674408, |
| "step": 292 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1452.2292175292969, |
| "epoch": 0.5021422450728363, |
| "grad_norm": 0.2539391815662384, |
| "kl": 0.001277923583984375, |
| "learning_rate": 4.93600044896063e-07, |
| "loss": 0.0331, |
| "reward": 0.2068037036806345, |
| "reward_std": 0.7527924031019211, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.24660906940698624, |
| "rewards/format_reward": 0.2812500074505806, |
| "step": 293 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1487.8125305175781, |
| "epoch": 0.5038560411311054, |
| "grad_norm": 0.28219902515411377, |
| "kl": 0.0015411376953125, |
| "learning_rate": 4.904846243842949e-07, |
| "loss": 0.0807, |
| "reward": 0.19793431088328362, |
| "reward_std": 0.5930087864398956, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.20344459637999535, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 294 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1381.0208740234375, |
| "epoch": 0.5055698371893744, |
| "grad_norm": 0.2740439176559448, |
| "kl": 0.002559661865234375, |
| "learning_rate": 4.873721045679706e-07, |
| "loss": 0.0127, |
| "reward": 0.519521371461451, |
| "reward_std": 0.8600037917494774, |
| "rewards/accuracy_reward": 0.06250000093132257, |
| "rewards/cosine_scaled_reward": -0.20514550060033798, |
| "rewards/format_reward": 0.2812500074505806, |
| "step": 295 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1623.0208740234375, |
| "epoch": 0.5072836332476436, |
| "grad_norm": 0.3007984161376953, |
| "kl": 0.0010242462158203125, |
| "learning_rate": 4.842626371469149e-07, |
| "loss": -0.007, |
| "reward": 0.4530015401542187, |
| "reward_std": 0.9436793178319931, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.20549920108169317, |
| "rewards/format_reward": 0.17708333767950535, |
| "step": 296 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1576.6250610351562, |
| "epoch": 0.5089974293059126, |
| "grad_norm": 0.2906862497329712, |
| "kl": 0.0011758804321289062, |
| "learning_rate": 4.811563736721829e-07, |
| "loss": 0.0599, |
| "reward": 0.49198414757847786, |
| "reward_std": 1.0118756145238876, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.18148974049836397, |
| "rewards/format_reward": 0.20833334140479565, |
| "step": 297 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1487.7083740234375, |
| "epoch": 0.5107112253641817, |
| "grad_norm": 0.352023720741272, |
| "kl": 0.00144195556640625, |
| "learning_rate": 4.780534655386743e-07, |
| "loss": -0.0307, |
| "reward": 0.37683416716754436, |
| "reward_std": 0.8631244823336601, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2052404135465622, |
| "rewards/format_reward": 0.2812500074505806, |
| "step": 298 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1460.5416870117188, |
| "epoch": 0.5124250214224507, |
| "grad_norm": 0.32392027974128723, |
| "kl": 0.00228118896484375, |
| "learning_rate": 4.749540639777539e-07, |
| "loss": -0.0335, |
| "reward": 0.6682421006262302, |
| "reward_std": 1.012038916349411, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.16558968648314476, |
| "rewards/format_reward": 0.3333333358168602, |
| "step": 299 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1429.0208740234375, |
| "epoch": 0.5141388174807198, |
| "grad_norm": 0.2622582018375397, |
| "kl": 0.0013561248779296875, |
| "learning_rate": 4.7185832004988133e-07, |
| "loss": 0.0258, |
| "reward": 0.8314435705542564, |
| "reward_std": 0.8176407516002655, |
| "rewards/accuracy_reward": 0.09375000186264515, |
| "rewards/cosine_scaled_reward": -0.13659141026437283, |
| "rewards/format_reward": 0.2291666716337204, |
| "step": 300 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1618.0208740234375, |
| "epoch": 0.5158526135389888, |
| "grad_norm": 0.33633002638816833, |
| "kl": 0.00131988525390625, |
| "learning_rate": 4.68766384637248e-07, |
| "loss": -0.0229, |
| "reward": 0.11395054124295712, |
| "reward_std": 0.571219764649868, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.24826429784297943, |
| "rewards/format_reward": 0.18750000558793545, |
| "step": 301 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1644.3958740234375, |
| "epoch": 0.517566409597258, |
| "grad_norm": 0.2794082462787628, |
| "kl": 0.0025787353515625, |
| "learning_rate": 4.656784084364238e-07, |
| "loss": 0.0632, |
| "reward": -0.022331595420837402, |
| "reward_std": 0.543972559273243, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2196408323943615, |
| "rewards/format_reward": 0.10416666977107525, |
| "step": 302 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1646.5625305175781, |
| "epoch": 0.519280205655527, |
| "grad_norm": 0.3117695748806, |
| "kl": 0.0012073516845703125, |
| "learning_rate": 4.6259454195101267e-07, |
| "loss": -0.0041, |
| "reward": 0.27055755932815373, |
| "reward_std": 0.7059797570109367, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2268887422978878, |
| "rewards/format_reward": 0.2187500037252903, |
| "step": 303 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1473.8333740234375, |
| "epoch": 0.5209940017137961, |
| "grad_norm": 0.25882959365844727, |
| "kl": 0.00136566162109375, |
| "learning_rate": 4.59514935484316e-07, |
| "loss": 0.0138, |
| "reward": 1.1540717035531998, |
| "reward_std": 0.7546401843428612, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.08102886518463492, |
| "rewards/format_reward": 0.2395833358168602, |
| "step": 304 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1562.8125305175781, |
| "epoch": 0.5227077977720651, |
| "grad_norm": 0.26582640409469604, |
| "kl": 0.0014476776123046875, |
| "learning_rate": 4.5643973913200837e-07, |
| "loss": -0.0198, |
| "reward": 0.4465249180793762, |
| "reward_std": 0.5320742838084698, |
| "rewards/accuracy_reward": 0.0520833358168602, |
| "rewards/cosine_scaled_reward": -0.18380979727953672, |
| "rewards/format_reward": 0.19791667722165585, |
| "step": 305 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1408.3750305175781, |
| "epoch": 0.5244215938303342, |
| "grad_norm": 0.26723894476890564, |
| "kl": 0.00182342529296875, |
| "learning_rate": 4.5336910277482155e-07, |
| "loss": 0.0754, |
| "reward": 0.5236281603574753, |
| "reward_std": 0.5826352834701538, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.19684619456529617, |
| "rewards/format_reward": 0.291666679084301, |
| "step": 306 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1548.9583740234375, |
| "epoch": 0.5261353898886033, |
| "grad_norm": 0.2609714865684509, |
| "kl": 0.0017099380493164062, |
| "learning_rate": 4.503031760712397e-07, |
| "loss": -0.0152, |
| "reward": 0.23132172971963882, |
| "reward_std": 0.7677483707666397, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.2582585848867893, |
| "rewards/format_reward": 0.2395833358168602, |
| "step": 307 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1513.5625305175781, |
| "epoch": 0.5278491859468724, |
| "grad_norm": 0.25365719199180603, |
| "kl": 0.0029115676879882812, |
| "learning_rate": 4.4724210845020494e-07, |
| "loss": -0.027, |
| "reward": 0.579220362007618, |
| "reward_std": 0.5990113839507103, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.17437524069100618, |
| "rewards/format_reward": 0.25000000931322575, |
| "step": 308 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1507.6042175292969, |
| "epoch": 0.5295629820051414, |
| "grad_norm": 0.31888437271118164, |
| "kl": 0.002178192138671875, |
| "learning_rate": 4.441860491038345e-07, |
| "loss": 0.0247, |
| "reward": 0.8379712477326393, |
| "reward_std": 1.0112811923027039, |
| "rewards/accuracy_reward": 0.09375000186264515, |
| "rewards/cosine_scaled_reward": -0.2099217213690281, |
| "rewards/format_reward": 0.3645833395421505, |
| "step": 309 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1639.1666870117188, |
| "epoch": 0.5312767780634104, |
| "grad_norm": 0.2766580283641815, |
| "kl": 0.001255035400390625, |
| "learning_rate": 4.4113514698014953e-07, |
| "loss": -0.023, |
| "reward": 0.06279195286333561, |
| "reward_std": 0.5707031115889549, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2568861898034811, |
| "rewards/format_reward": 0.1666666716337204, |
| "step": 310 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1546.6667175292969, |
| "epoch": 0.5329905741216795, |
| "grad_norm": 0.34961050748825073, |
| "kl": 0.001983642578125, |
| "learning_rate": 4.3808955077581546e-07, |
| "loss": 0.0053, |
| "reward": 0.027295149862766266, |
| "reward_std": 0.6128209084272385, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2780274897813797, |
| "rewards/format_reward": 0.21875000558793545, |
| "step": 311 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1541.3125305175781, |
| "epoch": 0.5347043701799485, |
| "grad_norm": 0.2850496172904968, |
| "kl": 0.0015439987182617188, |
| "learning_rate": 4.350494089288943e-07, |
| "loss": 0.041, |
| "reward": 0.895597904920578, |
| "reward_std": 0.996664509177208, |
| "rewards/accuracy_reward": 0.13541666697710752, |
| "rewards/cosine_scaled_reward": -0.09330026619136333, |
| "rewards/format_reward": 0.2916666716337204, |
| "step": 312 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1304.2708435058594, |
| "epoch": 0.5364181662382177, |
| "grad_norm": 0.34603336453437805, |
| "kl": 0.0025577545166015625, |
| "learning_rate": 4.3201486961161093e-07, |
| "loss": 0.0074, |
| "reward": 0.4570487830787897, |
| "reward_std": 0.7667558044195175, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.24745379388332367, |
| "rewards/format_reward": 0.3854166828095913, |
| "step": 313 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1478.7708740234375, |
| "epoch": 0.5381319622964867, |
| "grad_norm": 0.2592521905899048, |
| "kl": 0.0016460418701171875, |
| "learning_rate": 4.2898608072313045e-07, |
| "loss": 0.0546, |
| "reward": 0.4311096593737602, |
| "reward_std": 0.862842507660389, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.19942990876734257, |
| "rewards/format_reward": 0.19791667442768812, |
| "step": 314 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1462.8333740234375, |
| "epoch": 0.5398457583547558, |
| "grad_norm": 0.33211708068847656, |
| "kl": 0.0047512054443359375, |
| "learning_rate": 4.2596318988235037e-07, |
| "loss": -0.0538, |
| "reward": 0.3032863112166524, |
| "reward_std": 0.7379888445138931, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.1883798986673355, |
| "rewards/format_reward": 0.21875001303851604, |
| "step": 315 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1452.0625305175781, |
| "epoch": 0.5415595544130248, |
| "grad_norm": 0.27663302421569824, |
| "kl": 0.0022258758544921875, |
| "learning_rate": 4.2294634442070553e-07, |
| "loss": 0.0233, |
| "reward": 0.379562683403492, |
| "reward_std": 0.8121780008077621, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.22090492397546768, |
| "rewards/format_reward": 0.2916666744276881, |
| "step": 316 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1670.7083435058594, |
| "epoch": 0.5432733504712939, |
| "grad_norm": 0.25537002086639404, |
| "kl": 0.0017232894897460938, |
| "learning_rate": 4.1993569137498776e-07, |
| "loss": -0.0021, |
| "reward": -0.182917146012187, |
| "reward_std": 0.4011043682694435, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2771887518465519, |
| "rewards/format_reward": 0.10416666977107525, |
| "step": 317 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1319.6666870117188, |
| "epoch": 0.5449871465295629, |
| "grad_norm": 0.3191383481025696, |
| "kl": 0.002109527587890625, |
| "learning_rate": 4.1693137748017915e-07, |
| "loss": -0.0518, |
| "reward": 1.2622752636671066, |
| "reward_std": 0.6648239344358444, |
| "rewards/accuracy_reward": 0.13541667070239782, |
| "rewards/cosine_scaled_reward": -0.15025184489786625, |
| "rewards/format_reward": 0.3645833432674408, |
| "step": 318 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1539.3958435058594, |
| "epoch": 0.5467009425878321, |
| "grad_norm": 0.21360871195793152, |
| "kl": 0.001598358154296875, |
| "learning_rate": 4.1393354916230005e-07, |
| "loss": 0.0642, |
| "reward": 0.670879889279604, |
| "reward_std": 0.5511768870055676, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.17239019460976124, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 319 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1502.7500305175781, |
| "epoch": 0.5484147386461011, |
| "grad_norm": 0.30792051553726196, |
| "kl": 0.004299163818359375, |
| "learning_rate": 4.1094235253127374e-07, |
| "loss": 0.0965, |
| "reward": 0.5891358256340027, |
| "reward_std": 0.7148416712880135, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.21276572160422802, |
| "rewards/format_reward": 0.2395833358168602, |
| "step": 320 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1625.3542175292969, |
| "epoch": 0.5501285347043702, |
| "grad_norm": 0.2449854463338852, |
| "kl": 0.0019092559814453125, |
| "learning_rate": 4.079579333738039e-07, |
| "loss": 0.0027, |
| "reward": 0.09082813444547355, |
| "reward_std": 0.5155207589268684, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.23538977652788162, |
| "rewards/format_reward": 0.1666666679084301, |
| "step": 321 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1390.5625610351562, |
| "epoch": 0.5518423307626392, |
| "grad_norm": 0.29950419068336487, |
| "kl": 0.0020055770874023438, |
| "learning_rate": 4.0498043714627006e-07, |
| "loss": 0.0094, |
| "reward": 1.061479389667511, |
| "reward_std": 1.1913686990737915, |
| "rewards/accuracy_reward": 0.08333333674818277, |
| "rewards/cosine_scaled_reward": -0.1262189457193017, |
| "rewards/format_reward": 0.4583333507180214, |
| "step": 322 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1513.0208740234375, |
| "epoch": 0.5535561268209083, |
| "grad_norm": 0.2541593611240387, |
| "kl": 0.0019054412841796875, |
| "learning_rate": 4.020100089676376e-07, |
| "loss": 0.0179, |
| "reward": 0.48006347566843033, |
| "reward_std": 0.8878462538123131, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.24122651480138302, |
| "rewards/format_reward": 0.2500000074505806, |
| "step": 323 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1394.2917175292969, |
| "epoch": 0.5552699228791774, |
| "grad_norm": 0.28044670820236206, |
| "kl": 0.0023059844970703125, |
| "learning_rate": 3.9904679361238526e-07, |
| "loss": -0.0235, |
| "reward": 0.7225539498031139, |
| "reward_std": 0.8518542274832726, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.15590969566255808, |
| "rewards/format_reward": 0.36458334885537624, |
| "step": 324 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1490.8333740234375, |
| "epoch": 0.5569837189374465, |
| "grad_norm": 0.2727906405925751, |
| "kl": 0.00678253173828125, |
| "learning_rate": 3.9609093550344907e-07, |
| "loss": -0.0087, |
| "reward": -0.06165006849914789, |
| "reward_std": 0.5125335156917572, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.298244871199131, |
| "rewards/format_reward": 0.1770833358168602, |
| "step": 325 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1312.541732788086, |
| "epoch": 0.5586975149957155, |
| "grad_norm": 0.2801344394683838, |
| "kl": 0.002471923828125, |
| "learning_rate": 3.931425787051832e-07, |
| "loss": 0.0511, |
| "reward": 1.2787828594446182, |
| "reward_std": 0.8359841406345367, |
| "rewards/accuracy_reward": 0.14583333861082792, |
| "rewards/cosine_scaled_reward": -0.16505897510796785, |
| "rewards/format_reward": 0.4062500074505806, |
| "step": 326 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1458.1875610351562, |
| "epoch": 0.5604113110539846, |
| "grad_norm": 0.26847410202026367, |
| "kl": 0.001903533935546875, |
| "learning_rate": 3.902018669163384e-07, |
| "loss": -0.0135, |
| "reward": 0.42947175493463874, |
| "reward_std": 0.6873602792620659, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.2103552147746086, |
| "rewards/format_reward": 0.2500000037252903, |
| "step": 327 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1452.6875305175781, |
| "epoch": 0.5621251071122536, |
| "grad_norm": 0.3270891010761261, |
| "kl": 0.0029392242431640625, |
| "learning_rate": 3.872689434630585e-07, |
| "loss": 0.0059, |
| "reward": 0.6504845693707466, |
| "reward_std": 0.658838301897049, |
| "rewards/accuracy_reward": 0.09375000279396772, |
| "rewards/cosine_scaled_reward": -0.1178303137421608, |
| "rewards/format_reward": 0.31250002048909664, |
| "step": 328 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1420.5625305175781, |
| "epoch": 0.5638389031705227, |
| "grad_norm": 0.2690076529979706, |
| "kl": 0.0023937225341796875, |
| "learning_rate": 3.843439512918949e-07, |
| "loss": 0.0277, |
| "reward": 0.7830097079277039, |
| "reward_std": 1.0598141178488731, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.14712585415691137, |
| "rewards/format_reward": 0.3437500074505806, |
| "step": 329 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1569.0416870117188, |
| "epoch": 0.5655526992287918, |
| "grad_norm": 0.25364336371421814, |
| "kl": 0.0018033981323242188, |
| "learning_rate": 3.8142703296283953e-07, |
| "loss": 0.0415, |
| "reward": 0.6755895912647247, |
| "reward_std": 0.8362427651882172, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.17127279681153595, |
| "rewards/format_reward": 0.31250001583248377, |
| "step": 330 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1507.8333740234375, |
| "epoch": 0.5672664952870609, |
| "grad_norm": 0.2895563840866089, |
| "kl": 0.0018024444580078125, |
| "learning_rate": 3.785183306423767e-07, |
| "loss": -0.0391, |
| "reward": 0.2783973217010498, |
| "reward_std": 0.8718981444835663, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.24730180948972702, |
| "rewards/format_reward": 0.2500000037252903, |
| "step": 331 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1425.8750305175781, |
| "epoch": 0.5689802913453299, |
| "grad_norm": 0.34960538148880005, |
| "kl": 0.002040863037109375, |
| "learning_rate": 3.7561798609655373e-07, |
| "loss": -0.0507, |
| "reward": 0.4344450933858752, |
| "reward_std": 0.8201218247413635, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.20083919540047646, |
| "rewards/format_reward": 0.2395833395421505, |
| "step": 332 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1487.5209045410156, |
| "epoch": 0.570694087403599, |
| "grad_norm": 0.24745923280715942, |
| "kl": 0.0020313262939453125, |
| "learning_rate": 3.72726140684072e-07, |
| "loss": -0.0304, |
| "reward": 0.18487435579299927, |
| "reward_std": 0.5386057496070862, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.21758716367185116, |
| "rewards/format_reward": 0.22916666977107525, |
| "step": 333 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1550.2916870117188, |
| "epoch": 0.572407883461868, |
| "grad_norm": 0.26442721486091614, |
| "kl": 0.0026597976684570312, |
| "learning_rate": 3.6984293534939737e-07, |
| "loss": -0.0096, |
| "reward": 0.0944853127002716, |
| "reward_std": 0.6071152612566948, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2625133693218231, |
| "rewards/format_reward": 0.19791667256504297, |
| "step": 334 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1411.2291870117188, |
| "epoch": 0.5741216795201372, |
| "grad_norm": 0.26867979764938354, |
| "kl": 0.0038623809814453125, |
| "learning_rate": 3.6696851061588994e-07, |
| "loss": -0.0019, |
| "reward": 0.35343707352876663, |
| "reward_std": 0.7175738885998726, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2047328483313322, |
| "rewards/format_reward": 0.2708333386108279, |
| "step": 335 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1435.2708435058594, |
| "epoch": 0.5758354755784062, |
| "grad_norm": 0.2698570489883423, |
| "kl": 0.003559112548828125, |
| "learning_rate": 3.641030065789562e-07, |
| "loss": 0.0676, |
| "reward": 0.19872682355344296, |
| "reward_std": 0.5484345108270645, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2322644665837288, |
| "rewards/format_reward": 0.2395833395421505, |
| "step": 336 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1453.2500610351562, |
| "epoch": 0.5775492716366752, |
| "grad_norm": 0.39643779397010803, |
| "kl": 0.002422332763671875, |
| "learning_rate": 3.612465628992203e-07, |
| "loss": -0.0209, |
| "reward": 0.5667788721621037, |
| "reward_std": 0.9422286599874496, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.2142929658293724, |
| "rewards/format_reward": 0.2500000074505806, |
| "step": 337 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1499.2708435058594, |
| "epoch": 0.5792630676949443, |
| "grad_norm": 0.31420981884002686, |
| "kl": 0.0034742355346679688, |
| "learning_rate": 3.5839931879571725e-07, |
| "loss": -0.0471, |
| "reward": 0.6687855117488652, |
| "reward_std": 0.6993714831769466, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.17757828161120415, |
| "rewards/format_reward": 0.229166679084301, |
| "step": 338 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1635.0416870117188, |
| "epoch": 0.5809768637532133, |
| "grad_norm": 0.3503028452396393, |
| "kl": 0.002582550048828125, |
| "learning_rate": 3.555614130391079e-07, |
| "loss": 0.0312, |
| "reward": 0.11836901400238276, |
| "reward_std": 0.6972107961773872, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.25987397134304047, |
| "rewards/format_reward": 0.18750000558793545, |
| "step": 339 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1518.3125305175781, |
| "epoch": 0.5826906598114824, |
| "grad_norm": 0.2905596196651459, |
| "kl": 0.00196075439453125, |
| "learning_rate": 3.5273298394491515e-07, |
| "loss": 0.0276, |
| "reward": -0.018636616878211498, |
| "reward_std": 0.44363710284233093, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2682676538825035, |
| "rewards/format_reward": 0.229166679084301, |
| "step": 340 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1431.5833740234375, |
| "epoch": 0.5844044558697515, |
| "grad_norm": 0.38920703530311584, |
| "kl": 0.0020751953125, |
| "learning_rate": 3.4991416936678276e-07, |
| "loss": -0.081, |
| "reward": 0.17504378408193588, |
| "reward_std": 0.44530437886714935, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.24066567048430443, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 341 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1588.0833740234375, |
| "epoch": 0.5861182519280206, |
| "grad_norm": 0.2563221752643585, |
| "kl": 0.0012540817260742188, |
| "learning_rate": 3.471051066897562e-07, |
| "loss": -0.0366, |
| "reward": 0.24711184948682785, |
| "reward_std": 0.6161708235740662, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.24378404393792152, |
| "rewards/format_reward": 0.2708333432674408, |
| "step": 342 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1347.375015258789, |
| "epoch": 0.5878320479862896, |
| "grad_norm": 0.3236555755138397, |
| "kl": 0.002685546875, |
| "learning_rate": 3.4430593282358777e-07, |
| "loss": -0.0737, |
| "reward": 0.8499315101653337, |
| "reward_std": 0.7836231589317322, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.1686049527488649, |
| "rewards/format_reward": 0.4166666865348816, |
| "step": 343 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1482.1667175292969, |
| "epoch": 0.5895458440445587, |
| "grad_norm": 0.3159193694591522, |
| "kl": 0.002262115478515625, |
| "learning_rate": 3.4151678419606233e-07, |
| "loss": 0.0313, |
| "reward": 0.20483139622956514, |
| "reward_std": 0.6143444329500198, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.23227821104228497, |
| "rewards/format_reward": 0.2604166744276881, |
| "step": 344 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1510.5625610351562, |
| "epoch": 0.5912596401028277, |
| "grad_norm": 0.2599591016769409, |
| "kl": 0.0024280548095703125, |
| "learning_rate": 3.387377967463493e-07, |
| "loss": -0.1187, |
| "reward": 0.49281424283981323, |
| "reward_std": 0.833685964345932, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2351253591477871, |
| "rewards/format_reward": 0.3437500111758709, |
| "step": 345 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1518.3125, |
| "epoch": 0.5929734361610969, |
| "grad_norm": 0.2566784620285034, |
| "kl": 0.0026597976684570312, |
| "learning_rate": 3.359691059183761e-07, |
| "loss": 0.0576, |
| "reward": 0.10886371694505215, |
| "reward_std": 0.6810979098081589, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.25465162843465805, |
| "rewards/format_reward": 0.22916666977107525, |
| "step": 346 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1464.5834045410156, |
| "epoch": 0.5946872322193659, |
| "grad_norm": 0.3026113212108612, |
| "kl": 0.001850128173828125, |
| "learning_rate": 3.3321084665422803e-07, |
| "loss": -0.0558, |
| "reward": 0.26063048280775547, |
| "reward_std": 0.5907115042209625, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.26359253376722336, |
| "rewards/format_reward": 0.3437500149011612, |
| "step": 347 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1412.1875457763672, |
| "epoch": 0.596401028277635, |
| "grad_norm": 0.34688422083854675, |
| "kl": 0.00620269775390625, |
| "learning_rate": 3.3046315338757026e-07, |
| "loss": -0.0117, |
| "reward": 0.12697511911392212, |
| "reward_std": 0.4296252429485321, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2743792124092579, |
| "rewards/format_reward": 0.28125000558793545, |
| "step": 348 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1557.2500305175781, |
| "epoch": 0.598114824335904, |
| "grad_norm": 0.2776733636856079, |
| "kl": 0.00171661376953125, |
| "learning_rate": 3.2772616003709616e-07, |
| "loss": -0.0037, |
| "reward": 0.3554678609070834, |
| "reward_std": 0.9817065000534058, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.23372411355376244, |
| "rewards/format_reward": 0.20833333861082792, |
| "step": 349 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1624.9791870117188, |
| "epoch": 0.5998286203941731, |
| "grad_norm": 0.23979856073856354, |
| "kl": 0.0014801025390625, |
| "learning_rate": 3.250000000000001e-07, |
| "loss": -0.0343, |
| "reward": 0.3130597583949566, |
| "reward_std": 1.1155104637145996, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.24837487936019897, |
| "rewards/format_reward": 0.2291666716337204, |
| "step": 350 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1587.0625305175781, |
| "epoch": 0.6015424164524421, |
| "grad_norm": 0.2794056832790375, |
| "kl": 0.0021200180053710938, |
| "learning_rate": 3.222848061454764e-07, |
| "loss": -0.0059, |
| "reward": 0.17041855677962303, |
| "reward_std": 0.7474512904882431, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2524133399128914, |
| "rewards/format_reward": 0.20833333767950535, |
| "step": 351 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1703.5000305175781, |
| "epoch": 0.6032562125107113, |
| "grad_norm": 0.20891119539737701, |
| "kl": 0.00339508056640625, |
| "learning_rate": 3.195807108082429e-07, |
| "loss": 0.032, |
| "reward": -0.17395459115505219, |
| "reward_std": 0.34111763164401054, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.25343700870871544, |
| "rewards/format_reward": 0.08333333674818277, |
| "step": 352 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1624.5625305175781, |
| "epoch": 0.6049700085689803, |
| "grad_norm": 0.2826477885246277, |
| "kl": 0.001903533935546875, |
| "learning_rate": 3.168878457820915e-07, |
| "loss": 0.0494, |
| "reward": 0.03589773364365101, |
| "reward_std": 0.6567730531096458, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.25958870351314545, |
| "rewards/format_reward": 0.1875000037252903, |
| "step": 353 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1421.4375305175781, |
| "epoch": 0.6066838046272494, |
| "grad_norm": 0.2641639709472656, |
| "kl": 0.002178192138671875, |
| "learning_rate": 3.142063423134644e-07, |
| "loss": 0.0997, |
| "reward": 1.1788862720131874, |
| "reward_std": 0.7796865254640579, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.1469529727473855, |
| "rewards/format_reward": 0.4270833386108279, |
| "step": 354 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1403.5208740234375, |
| "epoch": 0.6083976006855184, |
| "grad_norm": 0.3317899703979492, |
| "kl": 0.002410888671875, |
| "learning_rate": 3.115363310950578e-07, |
| "loss": -0.0302, |
| "reward": 0.48206552490592003, |
| "reward_std": 0.5036356523633003, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.2182038016617298, |
| "rewards/format_reward": 0.2500000111758709, |
| "step": 355 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1523.6250305175781, |
| "epoch": 0.6101113967437874, |
| "grad_norm": 0.36468636989593506, |
| "kl": 0.0030536651611328125, |
| "learning_rate": 3.0887794225945143e-07, |
| "loss": -0.077, |
| "reward": 0.31238503009080887, |
| "reward_std": 0.8116755038499832, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.24230190180242062, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 356 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1531.3333740234375, |
| "epoch": 0.6118251928020566, |
| "grad_norm": 0.31790032982826233, |
| "kl": 0.00177764892578125, |
| "learning_rate": 3.062313053727671e-07, |
| "loss": -0.0039, |
| "reward": 0.07550723478198051, |
| "reward_std": 0.39915400743484497, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2813524901866913, |
| "rewards/format_reward": 0.2916666716337204, |
| "step": 357 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1640.6041870117188, |
| "epoch": 0.6135389888603257, |
| "grad_norm": 0.2601807415485382, |
| "kl": 0.0016021728515625, |
| "learning_rate": 3.0359654942835247e-07, |
| "loss": -0.0006, |
| "reward": 0.28700127080082893, |
| "reward_std": 1.0524575859308243, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.23822485655546188, |
| "rewards/format_reward": 0.17708333767950535, |
| "step": 358 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1546.6250305175781, |
| "epoch": 0.6152527849185947, |
| "grad_norm": 0.3116397559642792, |
| "kl": 0.003833770751953125, |
| "learning_rate": 3.0097380284049523e-07, |
| "loss": 0.0389, |
| "reward": 0.19815253466367722, |
| "reward_std": 0.7473566234111786, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.23686950653791428, |
| "rewards/format_reward": 0.18750000186264515, |
| "step": 359 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1337.1875305175781, |
| "epoch": 0.6169665809768637, |
| "grad_norm": 0.2909329831600189, |
| "kl": 0.0030670166015625, |
| "learning_rate": 2.9836319343816397e-07, |
| "loss": -0.0459, |
| "reward": 0.3552706688642502, |
| "reward_std": 0.7695464119315147, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.25929973274469376, |
| "rewards/format_reward": 0.2500000027939677, |
| "step": 360 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1522.6667175292969, |
| "epoch": 0.6186803770351328, |
| "grad_norm": 0.2280290722846985, |
| "kl": 0.002048492431640625, |
| "learning_rate": 2.9576484845877793e-07, |
| "loss": 0.0055, |
| "reward": 0.6520526558160782, |
| "reward_std": 1.0060482174158096, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.22033316269516945, |
| "rewards/format_reward": 0.2916666716337204, |
| "step": 361 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1703.8125305175781, |
| "epoch": 0.6203941730934018, |
| "grad_norm": 0.3011070191860199, |
| "kl": 0.001598358154296875, |
| "learning_rate": 2.931788945420058e-07, |
| "loss": -0.0108, |
| "reward": -0.06266473978757858, |
| "reward_std": 0.5431503728032112, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2694770060479641, |
| "rewards/format_reward": 0.11458333674818277, |
| "step": 362 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1513.3125305175781, |
| "epoch": 0.622107969151671, |
| "grad_norm": 0.3134266436100006, |
| "kl": 0.0020580291748046875, |
| "learning_rate": 2.9060545772359305e-07, |
| "loss": 0.038, |
| "reward": 0.2682766281068325, |
| "reward_std": 0.7535002008080482, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.19206186942756176, |
| "rewards/format_reward": 0.18750000558793545, |
| "step": 363 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1448.0417175292969, |
| "epoch": 0.62382176520994, |
| "grad_norm": 0.3311283588409424, |
| "kl": 0.0020236968994140625, |
| "learning_rate": 2.8804466342921987e-07, |
| "loss": -0.0098, |
| "reward": 0.15795885026454926, |
| "reward_std": 0.30675796419382095, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.1892208829522133, |
| "rewards/format_reward": 0.2500000074505806, |
| "step": 364 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1605.1667175292969, |
| "epoch": 0.6255355612682091, |
| "grad_norm": 0.36332038044929504, |
| "kl": 0.0021076202392578125, |
| "learning_rate": 2.854966364683872e-07, |
| "loss": 0.0331, |
| "reward": 0.1046671625226736, |
| "reward_std": 0.5455575659871101, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.20939945615828037, |
| "rewards/format_reward": 0.15625000838190317, |
| "step": 365 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1586.9791870117188, |
| "epoch": 0.6272493573264781, |
| "grad_norm": 0.29355624318122864, |
| "kl": 0.009174346923828125, |
| "learning_rate": 2.829615010283344e-07, |
| "loss": -0.027, |
| "reward": 0.5508327009156346, |
| "reward_std": 0.434384074062109, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.08403514325618744, |
| "rewards/format_reward": 0.1875000111758709, |
| "step": 366 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1256.0625305175781, |
| "epoch": 0.6289631533847472, |
| "grad_norm": 0.29369834065437317, |
| "kl": 0.003662109375, |
| "learning_rate": 2.8043938066798645e-07, |
| "loss": 0.0073, |
| "reward": 0.6286521656438708, |
| "reward_std": 0.9221112877130508, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.20299134403467178, |
| "rewards/format_reward": 0.3750000074505806, |
| "step": 367 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1618.8125610351562, |
| "epoch": 0.6306769494430163, |
| "grad_norm": 0.27679574489593506, |
| "kl": 0.0018301010131835938, |
| "learning_rate": 2.7793039831193133e-07, |
| "loss": 0.0365, |
| "reward": 0.1682092708069831, |
| "reward_std": 0.6755241900682449, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.24134992808103561, |
| "rewards/format_reward": 0.16666667349636555, |
| "step": 368 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1345.3542175292969, |
| "epoch": 0.6323907455012854, |
| "grad_norm": 0.2981966733932495, |
| "kl": 0.0034046173095703125, |
| "learning_rate": 2.7543467624442956e-07, |
| "loss": -0.0287, |
| "reward": 1.1676701717078686, |
| "reward_std": 1.2293783277273178, |
| "rewards/accuracy_reward": 0.12500000093132257, |
| "rewards/cosine_scaled_reward": -0.10941352834925056, |
| "rewards/format_reward": 0.35416667722165585, |
| "step": 369 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1419.2292022705078, |
| "epoch": 0.6341045415595544, |
| "grad_norm": 0.2704571485519409, |
| "kl": 0.0016345977783203125, |
| "learning_rate": 2.729523361034538e-07, |
| "loss": 0.0931, |
| "reward": 0.4965338706970215, |
| "reward_std": 0.559646487236023, |
| "rewards/accuracy_reward": 0.0520833358168602, |
| "rewards/cosine_scaled_reward": -0.2247798889875412, |
| "rewards/format_reward": 0.3125000074505806, |
| "step": 370 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1411.8541870117188, |
| "epoch": 0.6358183376178235, |
| "grad_norm": 0.3237265646457672, |
| "kl": 0.0018434524536132812, |
| "learning_rate": 2.7048349887476037e-07, |
| "loss": 0.0369, |
| "reward": 0.6508457884192467, |
| "reward_std": 0.8934041857719421, |
| "rewards/accuracy_reward": 0.06250000093132257, |
| "rewards/cosine_scaled_reward": -0.21806739643216133, |
| "rewards/format_reward": 0.3333333395421505, |
| "step": 371 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1625.5208740234375, |
| "epoch": 0.6375321336760925, |
| "grad_norm": 0.24745488166809082, |
| "kl": 0.002635955810546875, |
| "learning_rate": 2.6802828488599294e-07, |
| "loss": -0.0212, |
| "reward": 0.07880330551415682, |
| "reward_std": 0.5076464600861073, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.26656462997198105, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 372 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1609.5416870117188, |
| "epoch": 0.6392459297343616, |
| "grad_norm": 0.40871137380599976, |
| "kl": 0.0014896392822265625, |
| "learning_rate": 2.655868138008171e-07, |
| "loss": -0.0124, |
| "reward": 0.21406953036785126, |
| "reward_std": 0.6213681772351265, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.30080828070640564, |
| "rewards/format_reward": 0.2812500111758709, |
| "step": 373 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1501.5000305175781, |
| "epoch": 0.6409597257926307, |
| "grad_norm": 0.2766628563404083, |
| "kl": 0.002811431884765625, |
| "learning_rate": 2.631592046130896e-07, |
| "loss": -0.0421, |
| "reward": 0.3906727936118841, |
| "reward_std": 0.7104802057147026, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.2238925527781248, |
| "rewards/format_reward": 0.2500000111758709, |
| "step": 374 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1480.0625305175781, |
| "epoch": 0.6426735218508998, |
| "grad_norm": 0.3094847500324249, |
| "kl": 0.0052032470703125, |
| "learning_rate": 2.6074557564105724e-07, |
| "loss": -0.0191, |
| "reward": 0.15879609622061253, |
| "reward_std": 0.5000172480940819, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2149198092520237, |
| "rewards/format_reward": 0.2291666716337204, |
| "step": 375 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1327.0208740234375, |
| "epoch": 0.6443873179091688, |
| "grad_norm": 0.2825866639614105, |
| "kl": 0.00589752197265625, |
| "learning_rate": 2.583460445215911e-07, |
| "loss": -0.0239, |
| "reward": 0.5309350336901844, |
| "reward_std": 0.8217562958598137, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.2182630728930235, |
| "rewards/format_reward": 0.2708333432674408, |
| "step": 376 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1620.8333740234375, |
| "epoch": 0.6461011139674379, |
| "grad_norm": 0.2529444396495819, |
| "kl": 0.001277923583984375, |
| "learning_rate": 2.5596072820445254e-07, |
| "loss": -0.0177, |
| "reward": 0.24563428107649088, |
| "reward_std": 0.8564099371433258, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2212253138422966, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 377 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1585.9791870117188, |
| "epoch": 0.6478149100257069, |
| "grad_norm": 0.2812616527080536, |
| "kl": 0.002666473388671875, |
| "learning_rate": 2.5358974294659373e-07, |
| "loss": -0.0114, |
| "reward": 0.07188674993813038, |
| "reward_std": 0.8066864013671875, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.24227309226989746, |
| "rewards/format_reward": 0.14583333861082792, |
| "step": 378 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1236.2917175292969, |
| "epoch": 0.6495287060839761, |
| "grad_norm": 0.5107555985450745, |
| "kl": 0.0024261474609375, |
| "learning_rate": 2.512332043064913e-07, |
| "loss": -0.1417, |
| "reward": 0.7737401723861694, |
| "reward_std": 1.0626971647143364, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.20454938523471355, |
| "rewards/format_reward": 0.4583333358168602, |
| "step": 379 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1412.0208740234375, |
| "epoch": 0.6512425021422451, |
| "grad_norm": 0.5260189175605774, |
| "kl": 0.00237274169921875, |
| "learning_rate": 2.488912271385139e-07, |
| "loss": -0.0082, |
| "reward": 0.11665789876133204, |
| "reward_std": 0.7751671895384789, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.29444097727537155, |
| "rewards/format_reward": 0.2500000111758709, |
| "step": 380 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1650.5833740234375, |
| "epoch": 0.6529562982005142, |
| "grad_norm": 0.2257082313299179, |
| "kl": 0.002079010009765625, |
| "learning_rate": 2.465639255873246e-07, |
| "loss": -0.002, |
| "reward": -0.17455148696899414, |
| "reward_std": 0.3701091632246971, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2487034685909748, |
| "rewards/format_reward": 0.09375000186264515, |
| "step": 381 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1481.2083435058594, |
| "epoch": 0.6546700942587832, |
| "grad_norm": 0.30999627709388733, |
| "kl": 0.0020589828491210938, |
| "learning_rate": 2.4425141308231765e-07, |
| "loss": -0.0818, |
| "reward": 0.39649971574544907, |
| "reward_std": 0.8797931224107742, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.23154840990900993, |
| "rewards/format_reward": 0.20833333395421505, |
| "step": 382 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1477.8125305175781, |
| "epoch": 0.6563838903170522, |
| "grad_norm": 0.29673850536346436, |
| "kl": 0.00428009033203125, |
| "learning_rate": 2.4195380233209006e-07, |
| "loss": -0.0766, |
| "reward": 0.19455857016146183, |
| "reward_std": 0.541585698723793, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.26186586171388626, |
| "rewards/format_reward": 0.2708333358168602, |
| "step": 383 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1682.9583435058594, |
| "epoch": 0.6580976863753213, |
| "grad_norm": 0.22542622685432434, |
| "kl": 0.0011224746704101562, |
| "learning_rate": 2.3967120531894857e-07, |
| "loss": 0.0513, |
| "reward": -0.1359546771273017, |
| "reward_std": 0.35826287791132927, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2534148283302784, |
| "rewards/format_reward": 0.0937500037252903, |
| "step": 384 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1484.8541870117188, |
| "epoch": 0.6598114824335904, |
| "grad_norm": 0.27959033846855164, |
| "kl": 0.0029392242431640625, |
| "learning_rate": 2.374037332934512e-07, |
| "loss": -0.098, |
| "reward": -0.02571732923388481, |
| "reward_std": 0.44074755907058716, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2364521324634552, |
| "rewards/format_reward": 0.19791666977107525, |
| "step": 385 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1488.3958435058594, |
| "epoch": 0.6615252784918595, |
| "grad_norm": 0.2378430962562561, |
| "kl": 0.0012645721435546875, |
| "learning_rate": 2.3515149676898552e-07, |
| "loss": -0.039, |
| "reward": 0.07076568156480789, |
| "reward_std": 0.43841054663062096, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.24396153166890144, |
| "rewards/format_reward": 0.21875, |
| "step": 386 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1451.5208740234375, |
| "epoch": 0.6632390745501285, |
| "grad_norm": 0.3240017294883728, |
| "kl": 0.0023651123046875, |
| "learning_rate": 2.3291460551638237e-07, |
| "loss": 0.0978, |
| "reward": 0.6527014970779419, |
| "reward_std": 0.5769590213894844, |
| "rewards/accuracy_reward": 0.09375000279396772, |
| "rewards/cosine_scaled_reward": -0.21393180266022682, |
| "rewards/format_reward": 0.21875001024454832, |
| "step": 387 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1418.1042175292969, |
| "epoch": 0.6649528706083976, |
| "grad_norm": 0.2752251625061035, |
| "kl": 0.0028781890869140625, |
| "learning_rate": 2.306931685585657e-07, |
| "loss": -0.0529, |
| "reward": 0.8440244849771261, |
| "reward_std": 0.6565186530351639, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.2019278071820736, |
| "rewards/format_reward": 0.3854166753590107, |
| "step": 388 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1421.5208740234375, |
| "epoch": 0.6666666666666666, |
| "grad_norm": 0.27702751755714417, |
| "kl": 0.0026226043701171875, |
| "learning_rate": 2.2848729416523859e-07, |
| "loss": -0.0539, |
| "reward": 0.8581810109317303, |
| "reward_std": 0.5204750150442123, |
| "rewards/accuracy_reward": 0.1145833358168602, |
| "rewards/cosine_scaled_reward": -0.09280841797590256, |
| "rewards/format_reward": 0.2916666744276881, |
| "step": 389 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1378.8541870117188, |
| "epoch": 0.6683804627249358, |
| "grad_norm": 0.27244675159454346, |
| "kl": 0.0025920867919921875, |
| "learning_rate": 2.2629708984760706e-07, |
| "loss": -0.0489, |
| "reward": 0.8850560709834099, |
| "reward_std": 1.1492140740156174, |
| "rewards/accuracy_reward": 0.08333333395421505, |
| "rewards/cosine_scaled_reward": -0.20179678685963154, |
| "rewards/format_reward": 0.4270833432674408, |
| "step": 390 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1637.8542175292969, |
| "epoch": 0.6700942587832048, |
| "grad_norm": 0.27187544107437134, |
| "kl": 0.0049285888671875, |
| "learning_rate": 2.2412266235313973e-07, |
| "loss": 0.0146, |
| "reward": -0.018726075068116188, |
| "reward_std": 0.6156428083777428, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.274042472243309, |
| "rewards/format_reward": 0.13541666977107525, |
| "step": 391 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1484.9166717529297, |
| "epoch": 0.6718080548414739, |
| "grad_norm": 0.34059178829193115, |
| "kl": 0.002033233642578125, |
| "learning_rate": 2.2196411766036487e-07, |
| "loss": -0.0722, |
| "reward": 0.4430566392838955, |
| "reward_std": 0.6922470554709435, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.21358026936650276, |
| "rewards/format_reward": 0.28125000558793545, |
| "step": 392 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1628.9166870117188, |
| "epoch": 0.6735218508997429, |
| "grad_norm": 0.23434066772460938, |
| "kl": 0.00211334228515625, |
| "learning_rate": 2.1982156097370557e-07, |
| "loss": 0.0105, |
| "reward": -0.09393579512834549, |
| "reward_std": 0.3237507604062557, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.33932163566350937, |
| "rewards/format_reward": 0.1250000074505806, |
| "step": 393 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1315.3333740234375, |
| "epoch": 0.675235646958012, |
| "grad_norm": 0.29491373896598816, |
| "kl": 0.0037288665771484375, |
| "learning_rate": 2.1769509671835223e-07, |
| "loss": -0.0701, |
| "reward": 0.6508196443319321, |
| "reward_std": 0.8796055167913437, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.19033638876862824, |
| "rewards/format_reward": 0.3645833432674408, |
| "step": 394 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1452.7916870117188, |
| "epoch": 0.676949443016281, |
| "grad_norm": 0.27866828441619873, |
| "kl": 0.0017375946044921875, |
| "learning_rate": 2.1558482853517253e-07, |
| "loss": 0.0756, |
| "reward": 0.7415898907929659, |
| "reward_std": 0.6144783869385719, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.16752544231712818, |
| "rewards/format_reward": 0.3125, |
| "step": 395 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1655.9166870117188, |
| "epoch": 0.6786632390745502, |
| "grad_norm": 0.19680996239185333, |
| "kl": 0.0015344619750976562, |
| "learning_rate": 2.134908592756607e-07, |
| "loss": 0.0498, |
| "reward": 0.08432755898684263, |
| "reward_std": 0.5908685103058815, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2236979342997074, |
| "rewards/format_reward": 0.11458333674818277, |
| "step": 396 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1467.8125305175781, |
| "epoch": 0.6803770351328192, |
| "grad_norm": 0.2637212574481964, |
| "kl": 0.0015411376953125, |
| "learning_rate": 2.1141329099692406e-07, |
| "loss": -0.0323, |
| "reward": 0.3956471234560013, |
| "reward_std": 0.8686472177505493, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.22639739885926247, |
| "rewards/format_reward": 0.3020833358168602, |
| "step": 397 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1616.0209045410156, |
| "epoch": 0.6820908311910883, |
| "grad_norm": 0.2836722731590271, |
| "kl": 0.0020313262939453125, |
| "learning_rate": 2.0935222495670968e-07, |
| "loss": 0.0571, |
| "reward": 0.5134015195071697, |
| "reward_std": 0.7734205052256584, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.1734911259263754, |
| "rewards/format_reward": 0.16666666977107525, |
| "step": 398 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1232.1666870117188, |
| "epoch": 0.6838046272493573, |
| "grad_norm": 0.3226679861545563, |
| "kl": 0.004119873046875, |
| "learning_rate": 2.0730776160846853e-07, |
| "loss": -0.0493, |
| "reward": 1.2915575206279755, |
| "reward_std": 1.2753455489873886, |
| "rewards/accuracy_reward": 0.13541667349636555, |
| "rewards/cosine_scaled_reward": -0.15988415479660034, |
| "rewards/format_reward": 0.5208333507180214, |
| "step": 399 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1649.1667175292969, |
| "epoch": 0.6855184233076264, |
| "grad_norm": 0.2472459226846695, |
| "kl": 0.001773834228515625, |
| "learning_rate": 2.0528000059645995e-07, |
| "loss": -0.0103, |
| "reward": 0.06294973194599152, |
| "reward_std": 0.6322297528386116, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2695380821824074, |
| "rewards/format_reward": 0.15625000279396772, |
| "step": 400 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1584.5833740234375, |
| "epoch": 0.6872322193658955, |
| "grad_norm": 0.28003522753715515, |
| "kl": 0.002376556396484375, |
| "learning_rate": 2.032690407508949e-07, |
| "loss": 0.0172, |
| "reward": 0.29370687156915665, |
| "reward_std": 0.8225355744361877, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2329692356288433, |
| "rewards/format_reward": 0.20833333767950535, |
| "step": 401 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1528.4583435058594, |
| "epoch": 0.6889460154241646, |
| "grad_norm": 0.2969180643558502, |
| "kl": 0.0026397705078125, |
| "learning_rate": 2.0127498008311922e-07, |
| "loss": -0.0051, |
| "reward": -0.05280027166008949, |
| "reward_std": 0.3903113231062889, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2595442086458206, |
| "rewards/format_reward": 0.1770833358168602, |
| "step": 402 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1510.8333740234375, |
| "epoch": 0.6906598114824336, |
| "grad_norm": 0.26062414050102234, |
| "kl": 0.0029163360595703125, |
| "learning_rate": 1.9929791578083655e-07, |
| "loss": -0.0638, |
| "reward": 0.7391143590211868, |
| "reward_std": 0.758337527513504, |
| "rewards/accuracy_reward": 0.12500000279396772, |
| "rewards/cosine_scaled_reward": -0.13690827786922455, |
| "rewards/format_reward": 0.2500000037252903, |
| "step": 403 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1358.2917175292969, |
| "epoch": 0.6923736075407027, |
| "grad_norm": 0.3119449317455292, |
| "kl": 0.002368927001953125, |
| "learning_rate": 1.9733794420337213e-07, |
| "loss": -0.0557, |
| "reward": 0.17309055104851723, |
| "reward_std": 0.5400930494070053, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.28718583285808563, |
| "rewards/format_reward": 0.3541666679084301, |
| "step": 404 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1629.0208740234375, |
| "epoch": 0.6940874035989717, |
| "grad_norm": 0.23889358341693878, |
| "kl": 0.0018787384033203125, |
| "learning_rate": 1.9539516087697517e-07, |
| "loss": 0.0018, |
| "reward": 0.11960358917713165, |
| "reward_std": 0.7777920588850975, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.278524037450552, |
| "rewards/format_reward": 0.14583334140479565, |
| "step": 405 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1247.7083587646484, |
| "epoch": 0.6958011996572407, |
| "grad_norm": 0.36414635181427, |
| "kl": 0.003208160400390625, |
| "learning_rate": 1.934696604901642e-07, |
| "loss": 0.0489, |
| "reward": 0.6195550169795752, |
| "reward_std": 0.7279501482844353, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.23867519572377205, |
| "rewards/format_reward": 0.479166679084301, |
| "step": 406 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1667.6458435058594, |
| "epoch": 0.6975149957155099, |
| "grad_norm": 0.250451922416687, |
| "kl": 0.00220489501953125, |
| "learning_rate": 1.915615368891117e-07, |
| "loss": 0.0308, |
| "reward": 0.20595107227563858, |
| "reward_std": 0.48369817435741425, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.2192402444779873, |
| "rewards/format_reward": 0.1354166716337204, |
| "step": 407 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1319.9584045410156, |
| "epoch": 0.699228791773779, |
| "grad_norm": 0.27845609188079834, |
| "kl": 0.0021209716796875, |
| "learning_rate": 1.8967088307307e-07, |
| "loss": 0.1217, |
| "reward": 1.5674302279949188, |
| "reward_std": 1.2716419026255608, |
| "rewards/accuracy_reward": 0.16666667349636555, |
| "rewards/cosine_scaled_reward": -0.0756095617543906, |
| "rewards/format_reward": 0.3958333507180214, |
| "step": 408 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1454.5417175292969, |
| "epoch": 0.700942587832048, |
| "grad_norm": 0.3217247724533081, |
| "kl": 0.004810333251953125, |
| "learning_rate": 1.8779779118983867e-07, |
| "loss": 0.1388, |
| "reward": 0.5030364524573088, |
| "reward_std": 0.9113668613135815, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.20980912074446678, |
| "rewards/format_reward": 0.2500000074505806, |
| "step": 409 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1603.5625305175781, |
| "epoch": 0.702656383890317, |
| "grad_norm": 0.32288092374801636, |
| "kl": 0.002399444580078125, |
| "learning_rate": 1.8594235253127372e-07, |
| "loss": 0.0354, |
| "reward": 0.26620784075930715, |
| "reward_std": 0.5254248827695847, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.20196618512272835, |
| "rewards/format_reward": 0.20833334513008595, |
| "step": 410 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1488.3541870117188, |
| "epoch": 0.7043701799485861, |
| "grad_norm": 0.28620922565460205, |
| "kl": 0.0020618438720703125, |
| "learning_rate": 1.8410465752883758e-07, |
| "loss": 0.0085, |
| "reward": 0.3866404742002487, |
| "reward_std": 0.544437862932682, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.20075462386012077, |
| "rewards/format_reward": 0.3229166744276881, |
| "step": 411 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1503.9792175292969, |
| "epoch": 0.7060839760068551, |
| "grad_norm": 0.2854917347431183, |
| "kl": 0.002117156982421875, |
| "learning_rate": 1.822847957491922e-07, |
| "loss": -0.0722, |
| "reward": 0.17356615141034126, |
| "reward_std": 0.5067817345261574, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2639676071703434, |
| "rewards/format_reward": 0.3020833432674408, |
| "step": 412 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1527.3958435058594, |
| "epoch": 0.7077977720651243, |
| "grad_norm": 0.2824840843677521, |
| "kl": 0.0019626617431640625, |
| "learning_rate": 1.804828558898332e-07, |
| "loss": 0.1088, |
| "reward": 0.22076449799351394, |
| "reward_std": 0.5867825299501419, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.22575581818819046, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 413 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1580.7708740234375, |
| "epoch": 0.7095115681233933, |
| "grad_norm": 0.27595776319503784, |
| "kl": 0.0015163421630859375, |
| "learning_rate": 1.7869892577476722e-07, |
| "loss": -0.0465, |
| "reward": 0.38086844608187675, |
| "reward_std": 0.8873961195349693, |
| "rewards/accuracy_reward": 0.0520833358168602, |
| "rewards/cosine_scaled_reward": -0.21333472314290702, |
| "rewards/format_reward": 0.19791666977107525, |
| "step": 414 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1409.0000610351562, |
| "epoch": 0.7112253641816624, |
| "grad_norm": 0.28819894790649414, |
| "kl": 0.0022373199462890625, |
| "learning_rate": 1.7693309235023127e-07, |
| "loss": -0.129, |
| "reward": 0.3180142305791378, |
| "reward_std": 0.5801991075277328, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.23581846058368683, |
| "rewards/format_reward": 0.36458334885537624, |
| "step": 415 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1495.2500305175781, |
| "epoch": 0.7129391602399314, |
| "grad_norm": 0.375191867351532, |
| "kl": 0.003185272216796875, |
| "learning_rate": 1.7518544168045524e-07, |
| "loss": -0.0128, |
| "reward": 0.6192193161696196, |
| "reward_std": 0.7108025848865509, |
| "rewards/accuracy_reward": 0.08333333488553762, |
| "rewards/cosine_scaled_reward": -0.1999379526823759, |
| "rewards/format_reward": 0.22916666977107525, |
| "step": 416 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1419.6041870117188, |
| "epoch": 0.7146529562982005, |
| "grad_norm": 0.30786266922950745, |
| "kl": 0.0026340484619140625, |
| "learning_rate": 1.7345605894346726e-07, |
| "loss": 0.0226, |
| "reward": 0.545205045491457, |
| "reward_std": 0.9132278263568878, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.1991914976388216, |
| "rewards/format_reward": 0.30208334140479565, |
| "step": 417 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1369.125015258789, |
| "epoch": 0.7163667523564696, |
| "grad_norm": 0.34540000557899475, |
| "kl": 0.0025634765625, |
| "learning_rate": 1.7174502842694212e-07, |
| "loss": 0.0079, |
| "reward": 0.08001699112355709, |
| "reward_std": 0.37368160486221313, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2843298986554146, |
| "rewards/format_reward": 0.2916666753590107, |
| "step": 418 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1387.4583740234375, |
| "epoch": 0.7180805484147387, |
| "grad_norm": 0.339304119348526, |
| "kl": 0.011465072631835938, |
| "learning_rate": 1.7005243352409333e-07, |
| "loss": -0.0638, |
| "reward": 0.5061864908784628, |
| "reward_std": 0.7231926918029785, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.18234312906861305, |
| "rewards/format_reward": 0.3020833469927311, |
| "step": 419 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1450.6041870117188, |
| "epoch": 0.7197943444730077, |
| "grad_norm": 0.28949928283691406, |
| "kl": 0.0025787353515625, |
| "learning_rate": 1.6837835672960831e-07, |
| "loss": -0.0293, |
| "reward": 1.1634457781910896, |
| "reward_std": 0.8921896517276764, |
| "rewards/accuracy_reward": 0.11458333861082792, |
| "rewards/cosine_scaled_reward": -0.108680484816432, |
| "rewards/format_reward": 0.3333333469927311, |
| "step": 420 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1393.7709045410156, |
| "epoch": 0.7215081405312768, |
| "grad_norm": 0.33998945355415344, |
| "kl": 0.0024509429931640625, |
| "learning_rate": 1.6672287963562852e-07, |
| "loss": 0.0491, |
| "reward": 0.7930726930499077, |
| "reward_std": 0.9040541276335716, |
| "rewards/accuracy_reward": 0.06250000093132257, |
| "rewards/cosine_scaled_reward": -0.15836179442703724, |
| "rewards/format_reward": 0.322916679084301, |
| "step": 421 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1633.2916870117188, |
| "epoch": 0.7232219365895458, |
| "grad_norm": 0.23244516551494598, |
| "kl": 0.001995086669921875, |
| "learning_rate": 1.6508608292777203e-07, |
| "loss": 0.0037, |
| "reward": 0.10263706487603486, |
| "reward_std": 0.6201393157243729, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2293078862130642, |
| "rewards/format_reward": 0.17708334047347307, |
| "step": 422 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1548.1250305175781, |
| "epoch": 0.7249357326478149, |
| "grad_norm": 0.25799766182899475, |
| "kl": 0.0018787384033203125, |
| "learning_rate": 1.6346804638120098e-07, |
| "loss": -0.0096, |
| "reward": 0.36351461336016655, |
| "reward_std": 0.7988380044698715, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.18038328737020493, |
| "rewards/format_reward": 0.25000001303851604, |
| "step": 423 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1579.3958740234375, |
| "epoch": 0.726649528706084, |
| "grad_norm": 0.261885404586792, |
| "kl": 0.0017213821411132812, |
| "learning_rate": 1.6186884885673413e-07, |
| "loss": -0.0009, |
| "reward": 0.202106274664402, |
| "reward_std": 0.7989566773176193, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.19235052540898323, |
| "rewards/format_reward": 0.15625000558793545, |
| "step": 424 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1439.7291870117188, |
| "epoch": 0.7283633247643531, |
| "grad_norm": 0.2712852656841278, |
| "kl": 0.006488800048828125, |
| "learning_rate": 1.6028856829700258e-07, |
| "loss": 0.0239, |
| "reward": 0.5689312815666199, |
| "reward_std": 0.7213777899742126, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.18050884827971458, |
| "rewards/format_reward": 0.21875, |
| "step": 425 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1277.6250305175781, |
| "epoch": 0.7300771208226221, |
| "grad_norm": 0.49752435088157654, |
| "kl": 0.0027599334716796875, |
| "learning_rate": 1.5872728172265146e-07, |
| "loss": 0.076, |
| "reward": 1.1127668991684914, |
| "reward_std": 0.5749448239803314, |
| "rewards/accuracy_reward": 0.13541667349636555, |
| "rewards/cosine_scaled_reward": -0.14212442096322775, |
| "rewards/format_reward": 0.3020833386108279, |
| "step": 426 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1445.0, |
| "epoch": 0.7317909168808912, |
| "grad_norm": 0.25714749097824097, |
| "kl": 0.002696990966796875, |
| "learning_rate": 1.5718506522858572e-07, |
| "loss": -0.0134, |
| "reward": 0.10718778986483812, |
| "reward_std": 0.6740274652838707, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2864115312695503, |
| "rewards/format_reward": 0.2187500074505806, |
| "step": 427 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1624.6666870117188, |
| "epoch": 0.7335047129391602, |
| "grad_norm": 0.2530677318572998, |
| "kl": 0.0018749237060546875, |
| "learning_rate": 1.5566199398026147e-07, |
| "loss": 0.022, |
| "reward": 0.5430182013660669, |
| "reward_std": 0.6560900881886482, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.2140037715435028, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 428 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1622.2291870117188, |
| "epoch": 0.7352185089974294, |
| "grad_norm": 0.2503276765346527, |
| "kl": 0.002094268798828125, |
| "learning_rate": 1.5415814221002265e-07, |
| "loss": 0.0301, |
| "reward": -0.06853180006146431, |
| "reward_std": 0.37937263399362564, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2777785621583462, |
| "rewards/format_reward": 0.16666666697710752, |
| "step": 429 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1544.3750305175781, |
| "epoch": 0.7369323050556984, |
| "grad_norm": 0.250139981508255, |
| "kl": 0.0017986297607421875, |
| "learning_rate": 1.5267358321348285e-07, |
| "loss": 0.0141, |
| "reward": 0.6108243186026812, |
| "reward_std": 0.9471312090754509, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/cosine_scaled_reward": -0.17712798714637756, |
| "rewards/format_reward": 0.20833334140479565, |
| "step": 430 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1434.1041870117188, |
| "epoch": 0.7386461011139674, |
| "grad_norm": 0.28267359733581543, |
| "kl": 0.0019893646240234375, |
| "learning_rate": 1.5120838934595337e-07, |
| "loss": 0.0376, |
| "reward": 0.6465346980839968, |
| "reward_std": 1.2121013402938843, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.17127413116395473, |
| "rewards/format_reward": 0.3437500074505806, |
| "step": 431 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1608.9167175292969, |
| "epoch": 0.7403598971722365, |
| "grad_norm": 0.26306575536727905, |
| "kl": 0.0012712478637695312, |
| "learning_rate": 1.4976263201891613e-07, |
| "loss": 0.0194, |
| "reward": 0.3465429022908211, |
| "reward_std": 0.582036554813385, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.2412100676447153, |
| "rewards/format_reward": 0.14583333395421505, |
| "step": 432 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1531.3750305175781, |
| "epoch": 0.7420736932305055, |
| "grad_norm": 0.34143468737602234, |
| "kl": 0.0018463134765625, |
| "learning_rate": 1.483363816965435e-07, |
| "loss": 0.0579, |
| "reward": 0.4884794354438782, |
| "reward_std": 0.42937102913856506, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.13383641093969345, |
| "rewards/format_reward": 0.2604166669771075, |
| "step": 433 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1432.6250610351562, |
| "epoch": 0.7437874892887746, |
| "grad_norm": 0.38804128766059875, |
| "kl": 0.0039615631103515625, |
| "learning_rate": 1.469297078922642e-07, |
| "loss": -0.0035, |
| "reward": 0.42194677516818047, |
| "reward_std": 0.550103060901165, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.19477596133947372, |
| "rewards/format_reward": 0.3333333358168602, |
| "step": 434 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1563.6458435058594, |
| "epoch": 0.7455012853470437, |
| "grad_norm": 0.29043349623680115, |
| "kl": 0.001499176025390625, |
| "learning_rate": 1.4554267916537495e-07, |
| "loss": 0.0231, |
| "reward": 0.5839780867099762, |
| "reward_std": 0.7055636867880821, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/cosine_scaled_reward": -0.18507066927850246, |
| "rewards/format_reward": 0.2500000037252903, |
| "step": 435 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1419.2917175292969, |
| "epoch": 0.7472150814053128, |
| "grad_norm": 0.27879598736763, |
| "kl": 0.004342079162597656, |
| "learning_rate": 1.4417536311769885e-07, |
| "loss": 0.0289, |
| "reward": 0.8868619427084923, |
| "reward_std": 0.8792257979512215, |
| "rewards/accuracy_reward": 0.09375000279396772, |
| "rewards/cosine_scaled_reward": -0.1412948245415464, |
| "rewards/format_reward": 0.34375000558793545, |
| "step": 436 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1589.0208435058594, |
| "epoch": 0.7489288774635818, |
| "grad_norm": 0.2888442873954773, |
| "kl": 0.0017032623291015625, |
| "learning_rate": 1.4282782639029128e-07, |
| "loss": 0.0339, |
| "reward": 0.1717052087187767, |
| "reward_std": 0.6946492567658424, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2423543520271778, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 437 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1409.3958740234375, |
| "epoch": 0.7506426735218509, |
| "grad_norm": 0.2719264328479767, |
| "kl": 0.002471923828125, |
| "learning_rate": 1.4150013466019114e-07, |
| "loss": -0.0168, |
| "reward": 0.9626978933811188, |
| "reward_std": 0.7359797842800617, |
| "rewards/accuracy_reward": 0.12500000186264515, |
| "rewards/cosine_scaled_reward": -0.166676202788949, |
| "rewards/format_reward": 0.2708333432674408, |
| "step": 438 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1714.8750305175781, |
| "epoch": 0.7523564695801199, |
| "grad_norm": 0.24937574565410614, |
| "kl": 0.0011339187622070312, |
| "learning_rate": 1.4019235263722034e-07, |
| "loss": 0.0502, |
| "reward": -0.18545041419565678, |
| "reward_std": 0.2978869043290615, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.23207932710647583, |
| "rewards/format_reward": 0.052083334885537624, |
| "step": 439 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1352.2917022705078, |
| "epoch": 0.7540702656383891, |
| "grad_norm": 0.2822205722332001, |
| "kl": 0.0021915435791015625, |
| "learning_rate": 1.3890454406082956e-07, |
| "loss": -0.0195, |
| "reward": 0.6252860352396965, |
| "reward_std": 0.6529619544744492, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/cosine_scaled_reward": -0.20248195342719555, |
| "rewards/format_reward": 0.3333333423361182, |
| "step": 440 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1641.6875305175781, |
| "epoch": 0.7557840616966581, |
| "grad_norm": 0.2570679783821106, |
| "kl": 0.001667022705078125, |
| "learning_rate": 1.3763677169699217e-07, |
| "loss": -0.0778, |
| "reward": 0.003832288086414337, |
| "reward_std": 0.5638974718749523, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.25156646594405174, |
| "rewards/format_reward": 0.11458333861082792, |
| "step": 441 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1419.1666870117188, |
| "epoch": 0.7574978577549272, |
| "grad_norm": 0.2740538716316223, |
| "kl": 0.0019435882568359375, |
| "learning_rate": 1.3638909733514452e-07, |
| "loss": 0.0162, |
| "reward": 1.0481588691473007, |
| "reward_std": 0.7896323874592781, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.13111398927867413, |
| "rewards/format_reward": 0.4687500186264515, |
| "step": 442 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1333.5417175292969, |
| "epoch": 0.7592116538131962, |
| "grad_norm": 0.3648242950439453, |
| "kl": 0.0024566650390625, |
| "learning_rate": 1.351615817851748e-07, |
| "loss": 0.0397, |
| "reward": 0.47288158535957336, |
| "reward_std": 0.9505643546581268, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2298332378268242, |
| "rewards/format_reward": 0.3125000149011612, |
| "step": 443 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1656.5208740234375, |
| "epoch": 0.7609254498714653, |
| "grad_norm": 0.2964666783809662, |
| "kl": 0.0021820068359375, |
| "learning_rate": 1.3395428487445914e-07, |
| "loss": 0.0208, |
| "reward": 0.3792496621608734, |
| "reward_std": 0.8810505419969559, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.20247037336230278, |
| "rewards/format_reward": 0.25000000558793545, |
| "step": 444 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1681.6042175292969, |
| "epoch": 0.7626392459297343, |
| "grad_norm": 0.2914186418056488, |
| "kl": 0.00177001953125, |
| "learning_rate": 1.3276726544494571e-07, |
| "loss": 0.031, |
| "reward": 0.23477049777284265, |
| "reward_std": 0.5860272571444511, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/cosine_scaled_reward": -0.21530815958976746, |
| "rewards/format_reward": 0.1562500074505806, |
| "step": 445 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1518.3333435058594, |
| "epoch": 0.7643530419880035, |
| "grad_norm": 0.28216373920440674, |
| "kl": 0.001369476318359375, |
| "learning_rate": 1.316005813502869e-07, |
| "loss": 0.0654, |
| "reward": 0.6750286594033241, |
| "reward_std": 0.5412532538175583, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/cosine_scaled_reward": -0.15599552262574434, |
| "rewards/format_reward": 0.2708333432674408, |
| "step": 446 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1428.5208740234375, |
| "epoch": 0.7660668380462725, |
| "grad_norm": 0.30427539348602295, |
| "kl": 0.0028285980224609375, |
| "learning_rate": 1.3045428945301953e-07, |
| "loss": 0.0126, |
| "reward": 0.6193914785981178, |
| "reward_std": 0.9180549010634422, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/cosine_scaled_reward": -0.19425869919359684, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 447 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1671.0208740234375, |
| "epoch": 0.7677806341045416, |
| "grad_norm": 0.2518438398838043, |
| "kl": 0.0018138885498046875, |
| "learning_rate": 1.2932844562179352e-07, |
| "loss": 0.003, |
| "reward": 0.3041791720315814, |
| "reward_std": 0.8534430414438248, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.2137564681470394, |
| "rewards/format_reward": 0.21875000558793545, |
| "step": 448 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1607.8541870117188, |
| "epoch": 0.7694944301628106, |
| "grad_norm": 0.2538737952709198, |
| "kl": 0.0041255950927734375, |
| "learning_rate": 1.2822310472864885e-07, |
| "loss": 0.0294, |
| "reward": 0.08448375202715397, |
| "reward_std": 0.47953518480062485, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.2132504116743803, |
| "rewards/format_reward": 0.09375000186264515, |
| "step": 449 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1300.3541870117188, |
| "epoch": 0.7712082262210797, |
| "grad_norm": 0.282230406999588, |
| "kl": 0.0021991729736328125, |
| "learning_rate": 1.2713832064634125e-07, |
| "loss": 0.1019, |
| "reward": 0.7629837021231651, |
| "reward_std": 0.7489147633314133, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.15245907567441463, |
| "rewards/format_reward": 0.3020833358168602, |
| "step": 450 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1650.8750305175781, |
| "epoch": 0.7729220222793488, |
| "grad_norm": 0.2622946798801422, |
| "kl": 0.001873016357421875, |
| "learning_rate": 1.260741462457165e-07, |
| "loss": -0.016, |
| "reward": -0.12438017874956131, |
| "reward_std": 0.5109615363180637, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.28111129999160767, |
| "rewards/format_reward": 0.125, |
| "step": 451 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1508.0208740234375, |
| "epoch": 0.7746358183376179, |
| "grad_norm": 0.303348571062088, |
| "kl": 0.003978729248046875, |
| "learning_rate": 1.2503063339313356e-07, |
| "loss": -0.0728, |
| "reward": 0.7896851152181625, |
| "reward_std": 0.9815626963973045, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/cosine_scaled_reward": -0.17916808277368546, |
| "rewards/format_reward": 0.3437500149011612, |
| "step": 452 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1580.4583740234375, |
| "epoch": 0.7763496143958869, |
| "grad_norm": 0.2578333914279938, |
| "kl": 0.002101898193359375, |
| "learning_rate": 1.2400783294793668e-07, |
| "loss": -0.0507, |
| "reward": 0.3240011315792799, |
| "reward_std": 0.6389465481042862, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.17426074109971523, |
| "rewards/format_reward": 0.1041666679084301, |
| "step": 453 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1233.6250305175781, |
| "epoch": 0.778063410454156, |
| "grad_norm": 0.2885809540748596, |
| "kl": 0.00363922119140625, |
| "learning_rate": 1.2300579475997657e-07, |
| "loss": -0.0462, |
| "reward": 0.8891761973500252, |
| "reward_std": 1.197177767753601, |
| "rewards/accuracy_reward": 0.09375000186264515, |
| "rewards/cosine_scaled_reward": -0.19221091642975807, |
| "rewards/format_reward": 0.3645833432674408, |
| "step": 454 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1532.8542175292969, |
| "epoch": 0.779777206512425, |
| "grad_norm": 0.28073832392692566, |
| "kl": 0.0023908615112304688, |
| "learning_rate": 1.220245676671809e-07, |
| "loss": 0.0047, |
| "reward": 0.23808614909648895, |
| "reward_std": 0.5993708148598671, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.25339581072330475, |
| "rewards/format_reward": 0.2291666716337204, |
| "step": 455 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1470.7917175292969, |
| "epoch": 0.781491002570694, |
| "grad_norm": 0.2677416205406189, |
| "kl": 0.0023288726806640625, |
| "learning_rate": 1.2106419949317388e-07, |
| "loss": 0.0206, |
| "reward": 0.779910197481513, |
| "reward_std": 0.7993754521012306, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.17951993457973003, |
| "rewards/format_reward": 0.3020833395421505, |
| "step": 456 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1538.8541870117188, |
| "epoch": 0.7832047986289632, |
| "grad_norm": 0.3083825707435608, |
| "kl": 0.0019845962524414062, |
| "learning_rate": 1.2012473704494537e-07, |
| "loss": 0.0633, |
| "reward": 0.3929613158106804, |
| "reward_std": 0.4508157819509506, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.20179006457328796, |
| "rewards/format_reward": 0.14583333674818277, |
| "step": 457 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1352.1458740234375, |
| "epoch": 0.7849185946872322, |
| "grad_norm": 0.31749284267425537, |
| "kl": 0.00440216064453125, |
| "learning_rate": 1.1920622611056974e-07, |
| "loss": 0.0838, |
| "reward": 0.47300633788108826, |
| "reward_std": 1.0741110667586327, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.22252168133854866, |
| "rewards/format_reward": 0.3020833386108279, |
| "step": 458 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1414.8333740234375, |
| "epoch": 0.7866323907455013, |
| "grad_norm": 0.24783793091773987, |
| "kl": 0.0043487548828125, |
| "learning_rate": 1.1830871145697412e-07, |
| "loss": -0.0238, |
| "reward": 1.0414647981524467, |
| "reward_std": 0.9714280366897583, |
| "rewards/accuracy_reward": 0.09375000186264515, |
| "rewards/cosine_scaled_reward": -0.12982673710212111, |
| "rewards/format_reward": 0.3229166716337204, |
| "step": 459 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1585.8334045410156, |
| "epoch": 0.7883461868037703, |
| "grad_norm": 0.2753477394580841, |
| "kl": 0.001529693603515625, |
| "learning_rate": 1.1743223682775649e-07, |
| "loss": -0.0595, |
| "reward": -0.03680283762514591, |
| "reward_std": 0.5208316072821617, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.27455031499266624, |
| "rewards/format_reward": 0.1875000037252903, |
| "step": 460 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1480.3541870117188, |
| "epoch": 0.7900599828620394, |
| "grad_norm": 0.31823086738586426, |
| "kl": 0.002536773681640625, |
| "learning_rate": 1.1657684494105386e-07, |
| "loss": -0.0441, |
| "reward": 0.308930940926075, |
| "reward_std": 0.4995516464114189, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.18408312648534775, |
| "rewards/format_reward": 0.2812500074505806, |
| "step": 461 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1555.1041870117188, |
| "epoch": 0.7917737789203085, |
| "grad_norm": 0.27115893363952637, |
| "kl": 0.0023899078369140625, |
| "learning_rate": 1.1574257748745986e-07, |
| "loss": 0.0457, |
| "reward": 0.4185393452644348, |
| "reward_std": 0.8871691823005676, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.22503817081451416, |
| "rewards/format_reward": 0.28125001676380634, |
| "step": 462 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1492.3542175292969, |
| "epoch": 0.7934875749785776, |
| "grad_norm": 0.24115745723247528, |
| "kl": 0.0012664794921875, |
| "learning_rate": 1.1492947512799328e-07, |
| "loss": 0.0246, |
| "reward": 0.710046922788024, |
| "reward_std": 0.9779789745807648, |
| "rewards/accuracy_reward": 0.07291666977107525, |
| "rewards/cosine_scaled_reward": -0.17298575304448605, |
| "rewards/format_reward": 0.3020833460614085, |
| "step": 463 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1465.5000305175781, |
| "epoch": 0.7952013710368466, |
| "grad_norm": 0.3306126892566681, |
| "kl": 0.0021991729736328125, |
| "learning_rate": 1.1413757749211602e-07, |
| "loss": -0.0701, |
| "reward": 0.37903094850480556, |
| "reward_std": 0.9197189137339592, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.259335994720459, |
| "rewards/format_reward": 0.2916666753590107, |
| "step": 464 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1630.3333740234375, |
| "epoch": 0.7969151670951157, |
| "grad_norm": 0.30497926473617554, |
| "kl": 0.00186920166015625, |
| "learning_rate": 1.1336692317580158e-07, |
| "loss": 0.0361, |
| "reward": 0.021877296268939972, |
| "reward_std": 0.6427770256996155, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2640891894698143, |
| "rewards/format_reward": 0.14583333861082792, |
| "step": 465 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1671.7500305175781, |
| "epoch": 0.7986289631533847, |
| "grad_norm": 0.23499169945716858, |
| "kl": 0.001720428466796875, |
| "learning_rate": 1.1261754973965422e-07, |
| "loss": 0.0375, |
| "reward": -0.016727572306990623, |
| "reward_std": 0.6210906617343426, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.27923891320824623, |
| "rewards/format_reward": 0.12500000279396772, |
| "step": 466 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1690.8750305175781, |
| "epoch": 0.8003427592116538, |
| "grad_norm": 0.25364887714385986, |
| "kl": 0.0014820098876953125, |
| "learning_rate": 1.1188949370707787e-07, |
| "loss": 0.0486, |
| "reward": -0.025222748517990112, |
| "reward_std": 0.5080794095993042, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.2602507211267948, |
| "rewards/format_reward": 0.10416667070239782, |
| "step": 467 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1391.8750305175781, |
| "epoch": 0.8020565552699229, |
| "grad_norm": 0.3420558273792267, |
| "kl": 0.0025119781494140625, |
| "learning_rate": 1.1118279056249653e-07, |
| "loss": 0.0859, |
| "reward": 0.5714639741927385, |
| "reward_std": 0.7158600762486458, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.2035766988992691, |
| "rewards/format_reward": 0.28125000186264515, |
| "step": 468 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1536.4375610351562, |
| "epoch": 0.803770351328192, |
| "grad_norm": 0.2689160108566284, |
| "kl": 0.0021257400512695312, |
| "learning_rate": 1.1049747474962444e-07, |
| "loss": -0.012, |
| "reward": 0.007951326668262482, |
| "reward_std": 0.603579930961132, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.25903622433543205, |
| "rewards/format_reward": 0.18750000279396772, |
| "step": 469 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1389.8125305175781, |
| "epoch": 0.805484147386461, |
| "grad_norm": 0.2906559109687805, |
| "kl": 0.0021419525146484375, |
| "learning_rate": 1.0983357966978745e-07, |
| "loss": -0.0294, |
| "reward": 0.5438283756375313, |
| "reward_std": 1.0326615944504738, |
| "rewards/accuracy_reward": 0.06250000093132257, |
| "rewards/cosine_scaled_reward": -0.23368197679519653, |
| "rewards/format_reward": 0.3333333507180214, |
| "step": 470 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1593.9791870117188, |
| "epoch": 0.8071979434447301, |
| "grad_norm": 0.2347281128168106, |
| "kl": 0.0017852783203125, |
| "learning_rate": 1.0919113768029517e-07, |
| "loss": 0.0171, |
| "reward": -0.13019452383741736, |
| "reward_std": 0.42053942382335663, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.24632840603590012, |
| "rewards/format_reward": 0.12500000465661287, |
| "step": 471 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1487.3125305175781, |
| "epoch": 0.8089117395029991, |
| "grad_norm": 0.28390640020370483, |
| "kl": 0.003276824951171875, |
| "learning_rate": 1.0857018009286381e-07, |
| "loss": 0.0471, |
| "reward": 0.7808316666632891, |
| "reward_std": 0.8974563926458359, |
| "rewards/accuracy_reward": 0.08333333488553762, |
| "rewards/cosine_scaled_reward": -0.22984285652637482, |
| "rewards/format_reward": 0.3645833432674408, |
| "step": 472 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1613.0208740234375, |
| "epoch": 0.8106255355612683, |
| "grad_norm": 0.2611220180988312, |
| "kl": 0.00237274169921875, |
| "learning_rate": 1.0797073717209013e-07, |
| "loss": 0.0024, |
| "reward": 0.4201845917850733, |
| "reward_std": 0.5787086635828018, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.21403678506612778, |
| "rewards/format_reward": 0.1562500074505806, |
| "step": 473 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1424.4791717529297, |
| "epoch": 0.8123393316195373, |
| "grad_norm": 0.3259100615978241, |
| "kl": 0.003170013427734375, |
| "learning_rate": 1.0739283813397639e-07, |
| "loss": 0.0046, |
| "reward": 0.9525223551318049, |
| "reward_std": 0.773897759616375, |
| "rewards/accuracy_reward": 0.1041666679084301, |
| "rewards/cosine_scaled_reward": -0.1674602646380663, |
| "rewards/format_reward": 0.32291667722165585, |
| "step": 474 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1617.4583740234375, |
| "epoch": 0.8140531276778064, |
| "grad_norm": 0.27785417437553406, |
| "kl": 0.002117156982421875, |
| "learning_rate": 1.068365111445064e-07, |
| "loss": 0.002, |
| "reward": -0.034504614770412445, |
| "reward_std": 0.6453158259391785, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.29863275587558746, |
| "rewards/format_reward": 0.1458333395421505, |
| "step": 475 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1642.0208435058594, |
| "epoch": 0.8157669237360754, |
| "grad_norm": 0.24803964793682098, |
| "kl": 0.0032901763916015625, |
| "learning_rate": 1.063017833182728e-07, |
| "loss": -0.0068, |
| "reward": -0.033661461900919676, |
| "reward_std": 0.5855204090476036, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.27475955337285995, |
| "rewards/format_reward": 0.17708333767950535, |
| "step": 476 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1489.1875, |
| "epoch": 0.8174807197943444, |
| "grad_norm": 0.22895686328411102, |
| "kl": 0.0014104843139648438, |
| "learning_rate": 1.0578868071715544e-07, |
| "loss": 0.0127, |
| "reward": 0.5462241061031818, |
| "reward_std": 0.38443609699606895, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.23026614636182785, |
| "rewards/format_reward": 0.22916667349636555, |
| "step": 477 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1260.7708587646484, |
| "epoch": 0.8191945158526135, |
| "grad_norm": 0.36001700162887573, |
| "kl": 0.0029582977294921875, |
| "learning_rate": 1.0529722834905125e-07, |
| "loss": 0.0319, |
| "reward": 0.4286029487848282, |
| "reward_std": 0.7996908128261566, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.20781639032065868, |
| "rewards/format_reward": 0.3020833507180214, |
| "step": 478 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1482.6458740234375, |
| "epoch": 0.8209083119108826, |
| "grad_norm": 0.28881216049194336, |
| "kl": 0.0024471282958984375, |
| "learning_rate": 1.0482745016665526e-07, |
| "loss": 0.0271, |
| "reward": 0.5074643790721893, |
| "reward_std": 0.696723461151123, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/cosine_scaled_reward": -0.22283191978931427, |
| "rewards/format_reward": 0.2708333395421505, |
| "step": 479 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1449.2500305175781, |
| "epoch": 0.8226221079691517, |
| "grad_norm": 0.29771751165390015, |
| "kl": 0.0028934478759765625, |
| "learning_rate": 1.0437936906629334e-07, |
| "loss": -0.049, |
| "reward": 0.3999009942635894, |
| "reward_std": 0.8356301560997963, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.2523646242916584, |
| "rewards/format_reward": 0.2708333358168602, |
| "step": 480 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1517.0416870117188, |
| "epoch": 0.8243359040274207, |
| "grad_norm": 0.27192792296409607, |
| "kl": 0.0016155242919921875, |
| "learning_rate": 1.0395300688680625e-07, |
| "loss": -0.0018, |
| "reward": 0.2986976206302643, |
| "reward_std": 0.7875053882598877, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/cosine_scaled_reward": -0.27779635041952133, |
| "rewards/format_reward": 0.3229166716337204, |
| "step": 481 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1586.8958740234375, |
| "epoch": 0.8260497000856898, |
| "grad_norm": 0.23449064791202545, |
| "kl": 0.00141143798828125, |
| "learning_rate": 1.0354838440848501e-07, |
| "loss": 0.01, |
| "reward": 0.6149339154362679, |
| "reward_std": 0.32066725939512253, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/cosine_scaled_reward": -0.1879206132143736, |
| "rewards/format_reward": 0.22916666697710752, |
| "step": 482 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1352.1458740234375, |
| "epoch": 0.8277634961439588, |
| "grad_norm": 0.25975510478019714, |
| "kl": 0.002593994140625, |
| "learning_rate": 1.0316552135205837e-07, |
| "loss": 0.0456, |
| "reward": 0.8296075742691755, |
| "reward_std": 0.7724046930670738, |
| "rewards/accuracy_reward": 0.07291666883975267, |
| "rewards/cosine_scaled_reward": -0.14750460162758827, |
| "rewards/format_reward": 0.322916679084301, |
| "step": 483 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1492.1458435058594, |
| "epoch": 0.829477292202228, |
| "grad_norm": 0.3252848982810974, |
| "kl": 0.0024442672729492188, |
| "learning_rate": 1.0280443637773163e-07, |
| "loss": 0.0407, |
| "reward": 0.32957739010453224, |
| "reward_std": 0.9307611733675003, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.24090320616960526, |
| "rewards/format_reward": 0.2916666716337204, |
| "step": 484 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1334.6875305175781, |
| "epoch": 0.831191088260497, |
| "grad_norm": 0.3913592994213104, |
| "kl": 0.0066375732421875, |
| "learning_rate": 1.0246514708427701e-07, |
| "loss": -0.1766, |
| "reward": 0.49342756532132626, |
| "reward_std": 0.8096514493227005, |
| "rewards/accuracy_reward": 0.052083334885537624, |
| "rewards/cosine_scaled_reward": -0.22744118981063366, |
| "rewards/format_reward": 0.3541666744276881, |
| "step": 485 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1405.3542175292969, |
| "epoch": 0.8329048843187661, |
| "grad_norm": 0.3229379951953888, |
| "kl": 0.008695602416992188, |
| "learning_rate": 1.0214767000817596e-07, |
| "loss": 0.0611, |
| "reward": 0.6591449100524187, |
| "reward_std": 0.7455669045448303, |
| "rewards/accuracy_reward": 0.09375000279396772, |
| "rewards/cosine_scaled_reward": -0.1503998525440693, |
| "rewards/format_reward": 0.354166679084301, |
| "step": 486 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1569.8125, |
| "epoch": 0.8346186803770351, |
| "grad_norm": 0.3465595245361328, |
| "kl": 0.0024700164794921875, |
| "learning_rate": 1.0185202062281336e-07, |
| "loss": -0.1095, |
| "reward": -0.1328437142074108, |
| "reward_std": 0.28097547590732574, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.25740570947527885, |
| "rewards/format_reward": 0.12500000279396772, |
| "step": 487 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1341.8541870117188, |
| "epoch": 0.8363324764353042, |
| "grad_norm": 0.2883760631084442, |
| "kl": 0.0044956207275390625, |
| "learning_rate": 1.0157821333772304e-07, |
| "loss": -0.0132, |
| "reward": 0.8968268608077778, |
| "reward_std": 1.0606432855129242, |
| "rewards/accuracy_reward": 0.09375, |
| "rewards/cosine_scaled_reward": -0.1450992152094841, |
| "rewards/format_reward": 0.28125000558793545, |
| "step": 488 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1509.9375305175781, |
| "epoch": 0.8380462724935732, |
| "grad_norm": 0.3751949071884155, |
| "kl": 0.0017719268798828125, |
| "learning_rate": 1.013262614978859e-07, |
| "loss": -0.0342, |
| "reward": 0.4215831942856312, |
| "reward_std": 0.8841647803783417, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/cosine_scaled_reward": -0.21074516884982586, |
| "rewards/format_reward": 0.3020833348855376, |
| "step": 489 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1493.6875, |
| "epoch": 0.8397600685518424, |
| "grad_norm": 0.268597811460495, |
| "kl": 0.0024871826171875, |
| "learning_rate": 1.0109617738307911e-07, |
| "loss": 0.0308, |
| "reward": 0.3271262003108859, |
| "reward_std": 0.7303740531206131, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/cosine_scaled_reward": -0.2523518577218056, |
| "rewards/format_reward": 0.26041666977107525, |
| "step": 490 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1588.1667175292969, |
| "epoch": 0.8414738646101114, |
| "grad_norm": 0.2774799168109894, |
| "kl": 0.001842498779296875, |
| "learning_rate": 1.0088797220727779e-07, |
| "loss": -0.02, |
| "reward": -0.06127945396292489, |
| "reward_std": 0.4426092505455017, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2556958645582199, |
| "rewards/format_reward": 0.1770833358168602, |
| "step": 491 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1670.5833740234375, |
| "epoch": 0.8431876606683805, |
| "grad_norm": 0.25282374024391174, |
| "kl": 0.0024662017822265625, |
| "learning_rate": 1.0070165611810855e-07, |
| "loss": 0.0243, |
| "reward": 0.015509873628616333, |
| "reward_std": 0.5324227660894394, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/cosine_scaled_reward": -0.22523901239037514, |
| "rewards/format_reward": 0.1145833358168602, |
| "step": 492 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1433.6875305175781, |
| "epoch": 0.8449014567266495, |
| "grad_norm": 0.2538332939147949, |
| "kl": 0.0018558502197265625, |
| "learning_rate": 1.005372381963547e-07, |
| "loss": 0.0205, |
| "reward": 0.5890572015196085, |
| "reward_std": 0.7479087933897972, |
| "rewards/accuracy_reward": 0.0520833358168602, |
| "rewards/cosine_scaled_reward": -0.19769811630249023, |
| "rewards/format_reward": 0.322916679084301, |
| "step": 493 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1579.3333740234375, |
| "epoch": 0.8466152527849186, |
| "grad_norm": 0.2509302496910095, |
| "kl": 0.0020503997802734375, |
| "learning_rate": 1.0039472645551372e-07, |
| "loss": 0.0519, |
| "reward": 0.2534595690667629, |
| "reward_std": 1.029329240322113, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/cosine_scaled_reward": -0.2416459023952484, |
| "rewards/format_reward": 0.1666666716337204, |
| "step": 494 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1281.8750305175781, |
| "epoch": 0.8483290488431876, |
| "grad_norm": 0.3524729311466217, |
| "kl": 0.0039005279541015625, |
| "learning_rate": 1.002741278414069e-07, |
| "loss": -0.109, |
| "reward": 0.5718953423202038, |
| "reward_std": 0.9078854024410248, |
| "rewards/accuracy_reward": 0.05208333395421505, |
| "rewards/cosine_scaled_reward": -0.18542027287185192, |
| "rewards/format_reward": 0.34375000558793545, |
| "step": 495 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1470.7083740234375, |
| "epoch": 0.8500428449014568, |
| "grad_norm": 0.32555145025253296, |
| "kl": 0.0026264190673828125, |
| "learning_rate": 1.0017544823184055e-07, |
| "loss": 0.0229, |
| "reward": 0.5937455967068672, |
| "reward_std": 0.7896523922681808, |
| "rewards/accuracy_reward": 0.10416666697710752, |
| "rewards/cosine_scaled_reward": -0.11017215996980667, |
| "rewards/format_reward": 0.2500000037252903, |
| "step": 496 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1557.5833435058594, |
| "epoch": 0.8517566409597258, |
| "grad_norm": 0.29465195536613464, |
| "kl": 0.0020732879638671875, |
| "learning_rate": 1.0009869243631952e-07, |
| "loss": 0.0077, |
| "reward": 0.5861381143331528, |
| "reward_std": 1.3445016890764236, |
| "rewards/accuracy_reward": 0.06250000093132257, |
| "rewards/cosine_scaled_reward": -0.20600083097815514, |
| "rewards/format_reward": 0.2916666753590107, |
| "step": 497 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1610.7917175292969, |
| "epoch": 0.8534704370179949, |
| "grad_norm": 0.2673383951187134, |
| "kl": 0.001918792724609375, |
| "learning_rate": 1.000438641958131e-07, |
| "loss": 0.0054, |
| "reward": -0.13415055978111923, |
| "reward_std": 0.41844024509191513, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/cosine_scaled_reward": -0.2875858470797539, |
| "rewards/format_reward": 0.15625, |
| "step": 498 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1339.6458740234375, |
| "epoch": 0.8551842330762639, |
| "grad_norm": 0.29758375883102417, |
| "kl": 0.004123687744140625, |
| "learning_rate": 1.0001096618257236e-07, |
| "loss": 0.0713, |
| "reward": 0.968408640474081, |
| "reward_std": 1.159729391336441, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/cosine_scaled_reward": -0.16551226750016212, |
| "rewards/format_reward": 0.3333333395421505, |
| "step": 499 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 1332.2500305175781, |
| "epoch": 0.856898029134533, |
| "grad_norm": 0.28377655148506165, |
| "kl": 0.00530242919921875, |
| "learning_rate": 1e-07, |
| "loss": 0.1009, |
| "reward": 1.3940413501113653, |
| "reward_std": 1.01119065284729, |
| "rewards/accuracy_reward": 0.1770833395421505, |
| "rewards/cosine_scaled_reward": -0.041636522859334946, |
| "rewards/format_reward": 0.4270833507180214, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.856898029134533, |
| "step": 500, |
| "total_flos": 0.0, |
| "train_loss": 0.0032392389368615113, |
| "train_runtime": 46372.6786, |
| "train_samples_per_second": 0.518, |
| "train_steps_per_second": 0.011 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 500, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
