| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.009327923137913344, | |
| "eval_steps": 500, | |
| "global_step": 500, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 982.3750610351562, | |
| "epoch": 1.865584627582669e-05, | |
| "grad_norm": 0.06827232986688614, | |
| "kl": 0.0, | |
| "learning_rate": 3.3333333333333335e-07, | |
| "loss": 0.0163, | |
| "num_tokens": 39753.0, | |
| "reward": 0.2668055621907115, | |
| "reward_std": 0.258365576621145, | |
| "rewards/code_reward": 0.2638888955116272, | |
| "rewards/format_reward": 0.02916666679084301, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "epoch": 3.731169255165338e-05, | |
| "grad_norm": 0.06825702637434006, | |
| "kl": 0.0, | |
| "learning_rate": 6.666666666666667e-07, | |
| "loss": 0.0163, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.0010449413384776562, | |
| "epoch": 5.596753882748006e-05, | |
| "grad_norm": 0.06731275469064713, | |
| "kl": 0.00015044212341308594, | |
| "learning_rate": 1.0000000000000002e-06, | |
| "loss": 0.0165, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.0009167591924779117, | |
| "completion_length": 1241.1250610351562, | |
| "epoch": 7.462338510330675e-05, | |
| "grad_norm": 0.06577128916978836, | |
| "kl": 0.00014638900756835938, | |
| "learning_rate": 1.3333333333333334e-06, | |
| "loss": -0.0213, | |
| "num_tokens": 87072.0, | |
| "reward": 0.1686033308506012, | |
| "reward_std": 0.2493794783949852, | |
| "rewards/code_reward": 0.1686033308506012, | |
| "rewards/format_reward": 0.0, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0010596313513815403, | |
| "epoch": 9.327923137913344e-05, | |
| "grad_norm": 0.06366928666830063, | |
| "kl": 0.00015687942504882812, | |
| "learning_rate": 1.6666666666666667e-06, | |
| "loss": -0.0213, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.00101625599199906, | |
| "epoch": 0.00011193507765496012, | |
| "grad_norm": 0.06929203122854233, | |
| "kl": 0.00015020370483398438, | |
| "learning_rate": 2.0000000000000003e-06, | |
| "loss": -0.0213, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.0005019755335524678, | |
| "completion_length": 607.7500305175781, | |
| "epoch": 0.0001305909239307868, | |
| "grad_norm": 0.054728299379348755, | |
| "kl": 0.00021600723266601562, | |
| "learning_rate": 2.3333333333333336e-06, | |
| "loss": 0.0112, | |
| "num_tokens": 110922.0, | |
| "reward": 0.4583333432674408, | |
| "reward_std": 0.14433756470680237, | |
| "rewards/code_reward": 0.4583333432674408, | |
| "rewards/format_reward": 0.0, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.0005628917133435607, | |
| "epoch": 0.0001492467702066135, | |
| "grad_norm": 0.05518824979662895, | |
| "kl": 0.00023031234741210938, | |
| "learning_rate": 2.666666666666667e-06, | |
| "loss": 0.011, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0006851793150417507, | |
| "epoch": 0.00016790261648244018, | |
| "grad_norm": 0.05371546372771263, | |
| "kl": 0.00021839141845703125, | |
| "learning_rate": 3e-06, | |
| "loss": 0.0113, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.0009316161158494651, | |
| "completion_length": 763.25, | |
| "epoch": 0.00018655846275826687, | |
| "grad_norm": 0.06738901138305664, | |
| "kl": 0.00017595291137695312, | |
| "learning_rate": 3.3333333333333333e-06, | |
| "loss": -0.017, | |
| "num_tokens": 140040.0, | |
| "reward": 0.29250000417232513, | |
| "reward_std": 0.4735611826181412, | |
| "rewards/code_reward": 0.2916666716337204, | |
| "rewards/format_reward": 0.008333333767950535, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0005913192144362256, | |
| "epoch": 0.00020521430903409357, | |
| "grad_norm": 0.07004611194133759, | |
| "kl": 0.0001888275146484375, | |
| "learning_rate": 3.6666666666666666e-06, | |
| "loss": -0.0171, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.000849212781758979, | |
| "epoch": 0.00022387015530992023, | |
| "grad_norm": 0.069913350045681, | |
| "kl": 0.00017833709716796875, | |
| "learning_rate": 4.000000000000001e-06, | |
| "loss": -0.0172, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0005434846971184015, | |
| "completion_length": 622.1250305175781, | |
| "epoch": 0.00024252600158574693, | |
| "grad_norm": 0.061467599123716354, | |
| "kl": 0.00020265579223632812, | |
| "learning_rate": 4.333333333333334e-06, | |
| "loss": 0.0123, | |
| "num_tokens": 165519.0, | |
| "reward": 0.31522059440612793, | |
| "reward_std": 0.22002992033958435, | |
| "rewards/code_reward": 0.3139705955982208, | |
| "rewards/format_reward": 0.012500000186264515, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.0004921667859889567, | |
| "epoch": 0.0002611818478615736, | |
| "grad_norm": 0.0641019269824028, | |
| "kl": 0.000213623046875, | |
| "learning_rate": 4.666666666666667e-06, | |
| "loss": 0.0124, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.0007367254002019763, | |
| "epoch": 0.0002798376941374003, | |
| "grad_norm": 0.13394594192504883, | |
| "kl": 0.00019359588623046875, | |
| "learning_rate": 5e-06, | |
| "loss": 0.0122, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0015055734547786415, | |
| "completion_length": 809.2500305175781, | |
| "epoch": 0.000298493540413227, | |
| "grad_norm": 0.08201699703931808, | |
| "kl": 0.00023174285888671875, | |
| "learning_rate": 4.999952797253148e-06, | |
| "loss": 0.009, | |
| "num_tokens": 200625.0, | |
| "reward": 0.363928597420454, | |
| "reward_std": 0.2377874432131648, | |
| "rewards/code_reward": 0.3526785671710968, | |
| "rewards/format_reward": 0.11249998956918716, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0011205525370314717, | |
| "epoch": 0.0003171493866890537, | |
| "grad_norm": 0.08107728511095047, | |
| "kl": 0.00022745132446289062, | |
| "learning_rate": 4.9998111909931225e-06, | |
| "loss": 0.0088, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.0010780769807752222, | |
| "epoch": 0.00033580523296488035, | |
| "grad_norm": 0.08552956581115723, | |
| "kl": 0.00022220611572265625, | |
| "learning_rate": 4.999575187161439e-06, | |
| "loss": 0.0088, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.0009002784790936857, | |
| "completion_length": 777.0416870117188, | |
| "epoch": 0.0003544610792407071, | |
| "grad_norm": 0.08441054821014404, | |
| "kl": 0.00020885467529296875, | |
| "learning_rate": 4.9992447956603455e-06, | |
| "loss": -0.0108, | |
| "num_tokens": 234130.0, | |
| "reward": 0.6849345862865448, | |
| "reward_std": 0.30150339007377625, | |
| "rewards/code_reward": 0.6828512400388718, | |
| "rewards/format_reward": 0.02083333395421505, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.0011175140971317887, | |
| "epoch": 0.00037311692551653374, | |
| "grad_norm": 0.08343806117773056, | |
| "kl": 0.0002079010009765625, | |
| "learning_rate": 4.998820030352409e-06, | |
| "loss": -0.0108, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0009022637677844614, | |
| "epoch": 0.0003917727717923604, | |
| "grad_norm": 0.0833282396197319, | |
| "kl": 0.00020360946655273438, | |
| "learning_rate": 4.998300909059929e-06, | |
| "loss": -0.0111, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.0015675597824156284, | |
| "completion_length": 630.5833435058594, | |
| "epoch": 0.00041042861806818713, | |
| "grad_norm": 0.08936360478401184, | |
| "kl": 0.00019788742065429688, | |
| "learning_rate": 4.997687453564198e-06, | |
| "loss": 0.0142, | |
| "num_tokens": 257484.0, | |
| "reward": 0.3370833722874522, | |
| "reward_std": 0.252491211052984, | |
| "rewards/code_reward": 0.3333333432674408, | |
| "rewards/format_reward": 0.037500000558793545, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.001083480688976124, | |
| "epoch": 0.0004290844643440138, | |
| "grad_norm": 0.08784651011228561, | |
| "kl": 0.000202178955078125, | |
| "learning_rate": 4.9969796896045775e-06, | |
| "loss": 0.014, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.0011372591252438724, | |
| "epoch": 0.00044774031061984047, | |
| "grad_norm": 0.08748355507850647, | |
| "kl": 0.000209808349609375, | |
| "learning_rate": 4.996177646877426e-06, | |
| "loss": 0.014, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0014254440320655704, | |
| "completion_length": 834.6250305175781, | |
| "epoch": 0.0004663961568956672, | |
| "grad_norm": 0.07256177812814713, | |
| "kl": 0.00018978118896484375, | |
| "learning_rate": 4.995281359034851e-06, | |
| "loss": 0.0032, | |
| "num_tokens": 297543.0, | |
| "reward": 0.20448413118720055, | |
| "reward_std": 0.30803024023771286, | |
| "rewards/code_reward": 0.20198413357138634, | |
| "rewards/format_reward": 0.025000001303851604, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0013226708979345858, | |
| "epoch": 0.00048505200317149386, | |
| "grad_norm": 0.07238035649061203, | |
| "kl": 0.00017786026000976562, | |
| "learning_rate": 4.994290863683296e-06, | |
| "loss": 0.0033, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.0009347930317744613, | |
| "epoch": 0.0005037078494473206, | |
| "grad_norm": 0.07114218175411224, | |
| "kl": 0.00019788742065429688, | |
| "learning_rate": 4.99320620238196e-06, | |
| "loss": 0.0033, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.0008377690683118999, | |
| "completion_length": 899.4166870117188, | |
| "epoch": 0.0005223636957231473, | |
| "grad_norm": 0.12414078414440155, | |
| "kl": 0.00019073486328125, | |
| "learning_rate": 4.99202742064106e-06, | |
| "loss": 0.0106, | |
| "num_tokens": 331453.0, | |
| "reward": 0.15678573166951537, | |
| "reward_std": 0.17976204119622707, | |
| "rewards/code_reward": 0.1517857164144516, | |
| "rewards/format_reward": 0.05000000074505806, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0005996464460622519, | |
| "epoch": 0.0005410195419989739, | |
| "grad_norm": 0.07990814745426178, | |
| "kl": 0.00018262863159179688, | |
| "learning_rate": 4.990754567919917e-06, | |
| "loss": 0.0103, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.0008573587401770055, | |
| "epoch": 0.0005596753882748006, | |
| "grad_norm": 0.0788721814751625, | |
| "kl": 0.0001926422119140625, | |
| "learning_rate": 4.989387697624881e-06, | |
| "loss": 0.0104, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0007707980694249272, | |
| "completion_length": 873.875, | |
| "epoch": 0.0005783312345506273, | |
| "grad_norm": 0.07595472037792206, | |
| "kl": 0.0002765655517578125, | |
| "learning_rate": 4.987926867107095e-06, | |
| "loss": 0.0066, | |
| "num_tokens": 368998.0, | |
| "reward": 0.4440600574016571, | |
| "reward_std": 0.30831071734428406, | |
| "rewards/code_reward": 0.43614333122968674, | |
| "rewards/format_reward": 0.07916666753590107, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.0011201543966308236, | |
| "epoch": 0.000596987080826454, | |
| "grad_norm": 0.0760602131485939, | |
| "kl": 0.0002522468566894531, | |
| "learning_rate": 4.986372137660078e-06, | |
| "loss": 0.0069, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0011669989908114076, | |
| "epoch": 0.0006156429271022807, | |
| "grad_norm": 0.07646892964839935, | |
| "kl": 0.00030231475830078125, | |
| "learning_rate": 4.984723574517165e-06, | |
| "loss": 0.0063, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.0003657292982097715, | |
| "completion_length": 751.3333435058594, | |
| "epoch": 0.0006342987733781074, | |
| "grad_norm": 0.05740216001868248, | |
| "kl": 0.00026035308837890625, | |
| "learning_rate": 4.9829812468487655e-06, | |
| "loss": -0.0053, | |
| "num_tokens": 402006.0, | |
| "reward": 0.7125000059604645, | |
| "reward_std": 0.20762184262275696, | |
| "rewards/code_reward": 0.7083333283662796, | |
| "rewards/format_reward": 0.0416666679084301, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.00030940253054723144, | |
| "epoch": 0.000652954619653934, | |
| "grad_norm": 0.0546778179705143, | |
| "kl": 0.00029468536376953125, | |
| "learning_rate": 4.981145227759457e-06, | |
| "loss": -0.0054, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.00020408157433848828, | |
| "epoch": 0.0006716104659297607, | |
| "grad_norm": 0.05456351861357689, | |
| "kl": 0.0002956390380859375, | |
| "learning_rate": 4.979215594284924e-06, | |
| "loss": -0.0054, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0005608744686469436, | |
| "completion_length": 875.6667175292969, | |
| "epoch": 0.0006902663122055874, | |
| "grad_norm": 0.06957173347473145, | |
| "kl": 0.00028324127197265625, | |
| "learning_rate": 4.977192427388722e-06, | |
| "loss": 0.0157, | |
| "num_tokens": 444430.0, | |
| "reward": 0.515416668727994, | |
| "reward_std": 0.0217901524156332, | |
| "rewards/code_reward": 0.5, | |
| "rewards/format_reward": 0.15416665375232697, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.0005198217695578933, | |
| "epoch": 0.0007089221584814141, | |
| "grad_norm": 0.06720055639743805, | |
| "kl": 0.0002918243408203125, | |
| "learning_rate": 4.9750758119588824e-06, | |
| "loss": 0.0157, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.0003698411746881902, | |
| "epoch": 0.0007275780047572408, | |
| "grad_norm": 0.06269616633653641, | |
| "kl": 0.00034332275390625, | |
| "learning_rate": 4.972865836804349e-06, | |
| "loss": 0.0156, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.0008287752571050078, | |
| "completion_length": 668.0416870117188, | |
| "epoch": 0.0007462338510330675, | |
| "grad_norm": 0.10391846299171448, | |
| "kl": 0.00041675567626953125, | |
| "learning_rate": 4.970562594651254e-06, | |
| "loss": 0.066, | |
| "num_tokens": 469319.0, | |
| "reward": 0.3462119158357382, | |
| "reward_std": 0.24788545817136765, | |
| "rewards/code_reward": 0.32662858441472054, | |
| "rewards/format_reward": 0.1958333309739828, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0010252603678964078, | |
| "epoch": 0.0007648896973088942, | |
| "grad_norm": 0.10473919659852982, | |
| "kl": 0.00048065185546875, | |
| "learning_rate": 4.968166182139026e-06, | |
| "loss": 0.0657, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.0008386686386074871, | |
| "epoch": 0.0007835455435847208, | |
| "grad_norm": 0.10585241764783859, | |
| "kl": 0.0005102157592773438, | |
| "learning_rate": 4.9656766998163306e-06, | |
| "loss": 0.0657, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.0009505777561571449, | |
| "completion_length": 773.8750305175781, | |
| "epoch": 0.0008022013898605476, | |
| "grad_norm": 0.08152071386575699, | |
| "kl": 0.0006093978881835938, | |
| "learning_rate": 4.963094252136865e-06, | |
| "loss": 0.0317, | |
| "num_tokens": 500552.0, | |
| "reward": 0.46135416626930237, | |
| "reward_std": 0.2722689062356949, | |
| "rewards/code_reward": 0.44843751192092896, | |
| "rewards/format_reward": 0.12916667759418488, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.001228984270710498, | |
| "epoch": 0.0008208572361363743, | |
| "grad_norm": 0.08072768896818161, | |
| "kl": 0.0006775856018066406, | |
| "learning_rate": 4.960418947454958e-06, | |
| "loss": 0.0318, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.001452101394534111, | |
| "epoch": 0.0008395130824122009, | |
| "grad_norm": 0.07877793163061142, | |
| "kl": 0.0007939338684082031, | |
| "learning_rate": 4.957650898021038e-06, | |
| "loss": 0.0316, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0013901306956540793, | |
| "completion_length": 523.6666870117188, | |
| "epoch": 0.0008581689286880276, | |
| "grad_norm": 0.09457489103078842, | |
| "kl": 0.0012969970703125, | |
| "learning_rate": 4.954790219976915e-06, | |
| "loss": -0.0098, | |
| "num_tokens": 523440.0, | |
| "reward": 0.4345560818910599, | |
| "reward_std": 0.32685674726963043, | |
| "rewards/code_reward": 0.4170560836791992, | |
| "rewards/format_reward": 0.17500000074505806, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.0014728345850016922, | |
| "epoch": 0.0008768247749638543, | |
| "grad_norm": 0.09064733982086182, | |
| "kl": 0.00167083740234375, | |
| "learning_rate": 4.95183703335091e-06, | |
| "loss": -0.0102, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.0014083714631851763, | |
| "epoch": 0.0008954806212396809, | |
| "grad_norm": 0.09516453742980957, | |
| "kl": 0.002044677734375, | |
| "learning_rate": 4.948791462052819e-06, | |
| "loss": -0.0104, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0019310088246129453, | |
| "completion_length": 755.7083740234375, | |
| "epoch": 0.0009141364675155077, | |
| "grad_norm": 0.07378407567739487, | |
| "kl": 0.0006542205810546875, | |
| "learning_rate": 4.945653633868716e-06, | |
| "loss": 0.0031, | |
| "num_tokens": 552221.0, | |
| "reward": 0.5968180000782013, | |
| "reward_std": 0.36423343420028687, | |
| "rewards/code_reward": 0.5947346538305283, | |
| "rewards/format_reward": 0.02083333395421505, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.0012188139371573925, | |
| "epoch": 0.0009327923137913344, | |
| "grad_norm": 0.07792558521032333, | |
| "kl": 0.0006256103515625, | |
| "learning_rate": 4.942423680455584e-06, | |
| "loss": 0.0033, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0013152067258488387, | |
| "epoch": 0.000951448160067161, | |
| "grad_norm": 0.07279552519321442, | |
| "kl": 0.000690460205078125, | |
| "learning_rate": 4.939101737335802e-06, | |
| "loss": 0.003, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.0012177950120531023, | |
| "completion_length": 1433.25, | |
| "epoch": 0.0009701040063429877, | |
| "grad_norm": 0.0953638032078743, | |
| "kl": 0.0007143020629882812, | |
| "learning_rate": 4.935687943891447e-06, | |
| "loss": -0.0997, | |
| "num_tokens": 610451.0, | |
| "reward": 0.01875000074505806, | |
| "reward_std": 0.020104273222386837, | |
| "rewards/code_reward": 0.0, | |
| "rewards/format_reward": 0.1874999888241291, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0008826107368804514, | |
| "epoch": 0.0009887598526188144, | |
| "grad_norm": 0.07305457442998886, | |
| "kl": 0.0007915496826171875, | |
| "learning_rate": 4.932182443358458e-06, | |
| "loss": -0.0999, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.0009481910965405405, | |
| "epoch": 0.0010074156988946412, | |
| "grad_norm": 0.1027429923415184, | |
| "kl": 0.00098419189453125, | |
| "learning_rate": 4.928585382820616e-06, | |
| "loss": -0.0996, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.0013086027465760708, | |
| "completion_length": 899.5416870117188, | |
| "epoch": 0.0010260715451704677, | |
| "grad_norm": 0.07541598379611969, | |
| "kl": 0.00274658203125, | |
| "learning_rate": 4.924896913203376e-06, | |
| "loss": -0.0237, | |
| "num_tokens": 648072.0, | |
| "reward": 0.2216584151610732, | |
| "reward_std": 0.22969772201031446, | |
| "rewards/code_reward": 0.20915842056274414, | |
| "rewards/format_reward": 0.125, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.0009090554085560143, | |
| "epoch": 0.0010447273914462945, | |
| "grad_norm": 0.07311614602804184, | |
| "kl": 0.003662109375, | |
| "learning_rate": 4.921117189267535e-06, | |
| "loss": -0.0236, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0012668125564232469, | |
| "epoch": 0.0010633832377221213, | |
| "grad_norm": 0.07483425736427307, | |
| "kl": 0.00545501708984375, | |
| "learning_rate": 4.917246369602742e-06, | |
| "loss": -0.0237, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.0015675598988309503, | |
| "completion_length": 691.2916870117188, | |
| "epoch": 0.0010820390839979478, | |
| "grad_norm": 0.09265889972448349, | |
| "kl": 0.0068817138671875, | |
| "learning_rate": 4.9132846166208355e-06, | |
| "loss": -0.0068, | |
| "num_tokens": 678547.0, | |
| "reward": 0.2671003444120288, | |
| "reward_std": 0.2649538954719901, | |
| "rewards/code_reward": 0.2508503496646881, | |
| "rewards/format_reward": 0.16249999776482582, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.0011916961520910263, | |
| "epoch": 0.0011006949302737746, | |
| "grad_norm": 0.07879021763801575, | |
| "kl": 0.010772705078125, | |
| "learning_rate": 4.9092320965490365e-06, | |
| "loss": -0.0072, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.0013531986624002457, | |
| "epoch": 0.0011193507765496012, | |
| "grad_norm": 0.08185919374227524, | |
| "kl": 0.013824462890625, | |
| "learning_rate": 4.905088979422971e-06, | |
| "loss": -0.0072, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.001054943015333265, | |
| "completion_length": 816.0833740234375, | |
| "epoch": 0.001138006622825428, | |
| "grad_norm": 0.09033277630805969, | |
| "kl": 0.011138916015625, | |
| "learning_rate": 4.900855439079536e-06, | |
| "loss": 0.0434, | |
| "num_tokens": 708969.0, | |
| "reward": 0.13236112147569656, | |
| "reward_std": 0.17511780560016632, | |
| "rewards/code_reward": 0.1111111119389534, | |
| "rewards/format_reward": 0.21250000223517418, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.001191994990222156, | |
| "epoch": 0.0011566624691012545, | |
| "grad_norm": 0.11616651713848114, | |
| "kl": 0.0153656005859375, | |
| "learning_rate": 4.8965316531496055e-06, | |
| "loss": 0.0433, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.0013809394440613687, | |
| "epoch": 0.0011753183153770813, | |
| "grad_norm": 0.08028258383274078, | |
| "kl": 0.0198516845703125, | |
| "learning_rate": 4.892117803050578e-06, | |
| "loss": 0.0432, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.0014273470733314753, | |
| "completion_length": 478.12501525878906, | |
| "epoch": 0.001193974161652908, | |
| "grad_norm": 0.12240626662969589, | |
| "kl": 0.02911376953125, | |
| "learning_rate": 4.887614073978761e-06, | |
| "loss": 0.0251, | |
| "num_tokens": 727836.0, | |
| "reward": 0.5829170048236847, | |
| "reward_std": 0.42754843831062317, | |
| "rewards/code_reward": 0.5400003343820572, | |
| "rewards/format_reward": 0.42916665971279144, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.001149984309449792, | |
| "epoch": 0.0012126300079287346, | |
| "grad_norm": 0.11828020960092545, | |
| "kl": 0.0277099609375, | |
| "learning_rate": 4.883020654901609e-06, | |
| "loss": 0.0248, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.001388654694892466, | |
| "epoch": 0.0012312858542045614, | |
| "grad_norm": 0.15795664489269257, | |
| "kl": 0.02325439453125, | |
| "learning_rate": 4.878337738549785e-06, | |
| "loss": 0.0245, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.0012379828258417547, | |
| "completion_length": 1035.7083740234375, | |
| "epoch": 0.001249941700480388, | |
| "grad_norm": 0.07949040085077286, | |
| "kl": 0.0186614990234375, | |
| "learning_rate": 4.873565521409082e-06, | |
| "loss": 0.0077, | |
| "num_tokens": 767177.0, | |
| "reward": 0.07208333350718021, | |
| "reward_std": 0.15239802561700344, | |
| "rewards/code_reward": 0.0416666679084301, | |
| "rewards/format_reward": 0.30416667461395264, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.0013320732396095991, | |
| "epoch": 0.0012685975467562147, | |
| "grad_norm": 0.0713808611035347, | |
| "kl": 0.014373779296875, | |
| "learning_rate": 4.868704203712173e-06, | |
| "loss": 0.0076, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0018569266539998353, | |
| "epoch": 0.0012872533930320415, | |
| "grad_norm": 0.08412781357765198, | |
| "kl": 0.01141357421875, | |
| "learning_rate": 4.86375398943021e-06, | |
| "loss": 0.0076, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.0012622788199223578, | |
| "completion_length": 733.0416870117188, | |
| "epoch": 0.001305909239307868, | |
| "grad_norm": 0.08328709751367569, | |
| "kl": 0.01409912109375, | |
| "learning_rate": 4.858715086264274e-06, | |
| "loss": 0.0612, | |
| "num_tokens": 795078.0, | |
| "reward": 0.9353921413421631, | |
| "reward_std": 0.17414550855755806, | |
| "rewards/code_reward": 0.9053921699523926, | |
| "rewards/format_reward": 0.30000001192092896, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.001602432457730174, | |
| "epoch": 0.0013245650855836948, | |
| "grad_norm": 0.08351202309131622, | |
| "kl": 0.015625, | |
| "learning_rate": 4.853587705636646e-06, | |
| "loss": 0.0614, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.0013681039854418486, | |
| "epoch": 0.0013432209318595214, | |
| "grad_norm": 0.08263003826141357, | |
| "kl": 0.01739501953125, | |
| "learning_rate": 4.84837206268195e-06, | |
| "loss": 0.0613, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0006202208460308611, | |
| "completion_length": 748.2083435058594, | |
| "epoch": 0.0013618767781353482, | |
| "grad_norm": 0.08475489169359207, | |
| "kl": 0.01568603515625, | |
| "learning_rate": 4.8430683762381195e-06, | |
| "loss": 0.0156, | |
| "num_tokens": 823439.0, | |
| "reward": 0.46898147463798523, | |
| "reward_std": 0.3545081913471222, | |
| "rewards/code_reward": 0.4293981343507767, | |
| "rewards/format_reward": 0.3958333432674408, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.000859813007991761, | |
| "epoch": 0.0013805326244111747, | |
| "grad_norm": 0.08259416371583939, | |
| "kl": 0.016510009765625, | |
| "learning_rate": 4.837676868837213e-06, | |
| "loss": 0.0154, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.00038046142435632646, | |
| "epoch": 0.0013991884706870015, | |
| "grad_norm": 0.08513620495796204, | |
| "kl": 0.01690673828125, | |
| "learning_rate": 4.832197766696085e-06, | |
| "loss": 0.0151, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0009400486887898296, | |
| "completion_length": 746.0833435058594, | |
| "epoch": 0.0014178443169628283, | |
| "grad_norm": 0.08578525483608246, | |
| "kl": 0.009674072265625, | |
| "learning_rate": 4.826631299706887e-06, | |
| "loss": -0.0026, | |
| "num_tokens": 857425.0, | |
| "reward": 0.7170831263065338, | |
| "reward_std": 0.29470987617969513, | |
| "rewards/code_reward": 0.6824997961521149, | |
| "rewards/format_reward": 0.34583334624767303, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.000981206467258744, | |
| "epoch": 0.0014365001632386549, | |
| "grad_norm": 0.08752106130123138, | |
| "kl": 0.0097503662109375, | |
| "learning_rate": 4.820977701427424e-06, | |
| "loss": -0.0028, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.001091863785404712, | |
| "epoch": 0.0014551560095144816, | |
| "grad_norm": 0.08650875091552734, | |
| "kl": 0.010467529296875, | |
| "learning_rate": 4.81523720907136e-06, | |
| "loss": -0.0027, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.0008625364207546227, | |
| "completion_length": 843.0417175292969, | |
| "epoch": 0.0014738118557903082, | |
| "grad_norm": 0.08371932804584503, | |
| "kl": 0.01336669921875, | |
| "learning_rate": 4.809410063498254e-06, | |
| "loss": -0.0214, | |
| "num_tokens": 890714.0, | |
| "reward": 0.28875000309199095, | |
| "reward_std": 0.26341581624001265, | |
| "rewards/code_reward": 0.25, | |
| "rewards/format_reward": 0.38750001788139343, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.0010699788690544665, | |
| "epoch": 0.001492467702066135, | |
| "grad_norm": 0.08200695365667343, | |
| "kl": 0.0144500732421875, | |
| "learning_rate": 4.8034965092034656e-06, | |
| "loss": -0.0214, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.001069951627869159, | |
| "epoch": 0.0015111235483419617, | |
| "grad_norm": 0.08111701160669327, | |
| "kl": 0.013519287109375, | |
| "learning_rate": 4.797496794307889e-06, | |
| "loss": -0.0216, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.0005499664548551664, | |
| "completion_length": 688.2083435058594, | |
| "epoch": 0.0015297793946177883, | |
| "grad_norm": 0.09052547812461853, | |
| "kl": 0.0286865234375, | |
| "learning_rate": 4.791411170547545e-06, | |
| "loss": 0.0283, | |
| "num_tokens": 923131.0, | |
| "reward": 0.6477343738079071, | |
| "reward_std": 0.3908188194036484, | |
| "rewards/code_reward": 0.6152343600988388, | |
| "rewards/format_reward": 0.32500000298023224, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.0005515173252206296, | |
| "epoch": 0.001548435240893615, | |
| "grad_norm": 0.08914868533611298, | |
| "kl": 0.0316162109375, | |
| "learning_rate": 4.785239893263017e-06, | |
| "loss": 0.0282, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.0004736810806207359, | |
| "epoch": 0.0015670910871694416, | |
| "grad_norm": 0.08998391777276993, | |
| "kl": 0.0325927734375, | |
| "learning_rate": 4.778983221388742e-06, | |
| "loss": 0.0282, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0016970738070085645, | |
| "completion_length": 1049.75, | |
| "epoch": 0.0015857469334452684, | |
| "grad_norm": 0.07974597066640854, | |
| "kl": 0.0087890625, | |
| "learning_rate": 4.77264141744214e-06, | |
| "loss": -0.0355, | |
| "num_tokens": 968365.0, | |
| "reward": 0.19875000324100256, | |
| "reward_std": 0.2349306456744671, | |
| "rewards/code_reward": 0.1666666716337204, | |
| "rewards/format_reward": 0.32083334028720856, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.001736409030854702, | |
| "epoch": 0.0016044027797210952, | |
| "grad_norm": 0.0792006105184555, | |
| "kl": 0.009368896484375, | |
| "learning_rate": 4.766214747512603e-06, | |
| "loss": -0.0355, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.0015154886059463024, | |
| "epoch": 0.0016230586259969218, | |
| "grad_norm": 0.10796553641557693, | |
| "kl": 0.009796142578125, | |
| "learning_rate": 4.759703481250331e-06, | |
| "loss": -0.036, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.0013903837534599006, | |
| "completion_length": 792.4583435058594, | |
| "epoch": 0.0016417144722727485, | |
| "grad_norm": 0.19102805852890015, | |
| "kl": 0.0703125, | |
| "learning_rate": 4.753107891855015e-06, | |
| "loss": 0.0435, | |
| "num_tokens": 999360.0, | |
| "reward": 0.3778717517852783, | |
| "reward_std": 0.32771627604961395, | |
| "rewards/code_reward": 0.3512050434947014, | |
| "rewards/format_reward": 0.2666666656732559, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0016154100885614753, | |
| "epoch": 0.001660370318548575, | |
| "grad_norm": 0.09603425115346909, | |
| "kl": 0.0440673828125, | |
| "learning_rate": 4.746428256064375e-06, | |
| "loss": 0.0434, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.0018210893031209707, | |
| "epoch": 0.0016790261648244019, | |
| "grad_norm": 0.09434056282043457, | |
| "kl": 0.0350341796875, | |
| "learning_rate": 4.7396648541425534e-06, | |
| "loss": 0.0433, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0014776080497540534, | |
| "completion_length": 766.6666870117188, | |
| "epoch": 0.0016976820111002284, | |
| "grad_norm": 0.08751969784498215, | |
| "kl": 0.012847900390625, | |
| "learning_rate": 4.732817969868348e-06, | |
| "loss": -0.0055, | |
| "num_tokens": 1032760.0, | |
| "reward": 0.503094419836998, | |
| "reward_std": 0.4136483669281006, | |
| "rewards/code_reward": 0.4914277046918869, | |
| "rewards/format_reward": 0.11666666716337204, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_ratio": 0.0013046148815192282, | |
| "epoch": 0.0017163378573760552, | |
| "grad_norm": 0.08883315324783325, | |
| "kl": 0.010772705078125, | |
| "learning_rate": 4.7258878905233095e-06, | |
| "loss": -0.0056, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_ratio": 0.000994603120489046, | |
| "epoch": 0.001734993703651882, | |
| "grad_norm": 0.10364013910293579, | |
| "kl": 0.010162353515625, | |
| "learning_rate": 4.718874906879688e-06, | |
| "loss": -0.0057, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_ratio": 0.0011711710831150413, | |
| "completion_length": 1132.4583740234375, | |
| "epoch": 0.0017536495499277085, | |
| "grad_norm": 0.07417519390583038, | |
| "kl": 0.0053558349609375, | |
| "learning_rate": 4.711779313188231e-06, | |
| "loss": 0.0018, | |
| "num_tokens": 1080639.0, | |
| "reward": 0.04583333432674408, | |
| "reward_std": 0.026794558390975, | |
| "rewards/code_reward": 0.0, | |
| "rewards/format_reward": 0.4583333432674408, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_ratio": 0.00107975909486413, | |
| "epoch": 0.0017723053962035353, | |
| "grad_norm": 0.07358817011117935, | |
| "kl": 0.005706787109375, | |
| "learning_rate": 4.70460140716584e-06, | |
| "loss": 0.0019, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0008912903140299022, | |
| "epoch": 0.0017909612424793619, | |
| "grad_norm": 0.08615133166313171, | |
| "kl": 0.0057830810546875, | |
| "learning_rate": 4.697341489983076e-06, | |
| "loss": 0.0019, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_ratio": 0.002020051470026374, | |
| "completion_length": 727.7500305175781, | |
| "epoch": 0.0018096170887551887, | |
| "grad_norm": 0.0899001881480217, | |
| "kl": 0.01202392578125, | |
| "learning_rate": 4.6899998662515215e-06, | |
| "loss": -0.001, | |
| "num_tokens": 1111077.0, | |
| "reward": 0.29144802689552307, | |
| "reward_std": 0.33243465423583984, | |
| "rewards/code_reward": 0.2614480257034302, | |
| "rewards/format_reward": 0.30000001192092896, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_ratio": 0.0011579456622712314, | |
| "epoch": 0.0018282729350310154, | |
| "grad_norm": 0.09250804036855698, | |
| "kl": 0.013916015625, | |
| "learning_rate": 4.682576844011007e-06, | |
| "loss": -0.0011, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_ratio": 0.0017355763702653348, | |
| "epoch": 0.001846928781306842, | |
| "grad_norm": 0.0963953509926796, | |
| "kl": 0.015106201171875, | |
| "learning_rate": 4.675072734716678e-06, | |
| "loss": -0.0013, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_ratio": 0.0009290422021877021, | |
| "completion_length": 1036.0833740234375, | |
| "epoch": 0.0018655846275826688, | |
| "grad_norm": 0.09167575091123581, | |
| "kl": 0.015167236328125, | |
| "learning_rate": 4.667487853225931e-06, | |
| "loss": -0.0103, | |
| "num_tokens": 1155731.0, | |
| "reward": 0.11254538409411907, | |
| "reward_std": 0.16347683407366276, | |
| "rewards/code_reward": 0.07879538089036942, | |
| "rewards/format_reward": 0.3374999910593033, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0006831295904703438, | |
| "epoch": 0.0018842404738584953, | |
| "grad_norm": 0.08466409146785736, | |
| "kl": 0.016815185546875, | |
| "learning_rate": 4.659822517785203e-06, | |
| "loss": -0.0103, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_ratio": 0.000901765946764499, | |
| "epoch": 0.001902896320134322, | |
| "grad_norm": 0.09180773049592972, | |
| "kl": 0.017242431640625, | |
| "learning_rate": 4.6520770500166165e-06, | |
| "loss": -0.0105, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_ratio": 0.0008878828666638583, | |
| "completion_length": 771.9583435058594, | |
| "epoch": 0.0019215521664101489, | |
| "grad_norm": 0.11708436161279678, | |
| "kl": 0.029144287109375, | |
| "learning_rate": 4.644251774904487e-06, | |
| "loss": 0.0148, | |
| "num_tokens": 1188838.0, | |
| "reward": 0.03181159868836403, | |
| "reward_std": 0.02184708882123232, | |
| "rewards/code_reward": 0.0018115942366421223, | |
| "rewards/format_reward": 0.29999998211860657, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_ratio": 0.0006120741018094122, | |
| "epoch": 0.0019402080126859754, | |
| "grad_norm": 0.10810239613056183, | |
| "kl": 0.03369140625, | |
| "learning_rate": 4.636347020781684e-06, | |
| "loss": 0.0148, | |
| "step": 104 | |
| }, | |
| { | |
| "clip_ratio": 0.0008727995736990124, | |
| "epoch": 0.0019588638589618022, | |
| "grad_norm": 0.11280914396047592, | |
| "kl": 0.0435791015625, | |
| "learning_rate": 4.6283631193158605e-06, | |
| "loss": 0.0145, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.0009058912692125887, | |
| "completion_length": 914.5417175292969, | |
| "epoch": 0.0019775197052376288, | |
| "grad_norm": 0.07587506622076035, | |
| "kl": 0.0174560546875, | |
| "learning_rate": 4.620300405495532e-06, | |
| "loss": 0.0523, | |
| "num_tokens": 1222319.0, | |
| "reward": 0.15707794576883316, | |
| "reward_std": 0.2763161137700081, | |
| "rewards/code_reward": 0.12624460831284523, | |
| "rewards/format_reward": 0.3083333447575569, | |
| "step": 106 | |
| }, | |
| { | |
| "clip_ratio": 0.0009446362673770636, | |
| "epoch": 0.0019961755515134553, | |
| "grad_norm": 0.07568401843309402, | |
| "kl": 0.01898193359375, | |
| "learning_rate": 4.612159217616022e-06, | |
| "loss": 0.0523, | |
| "step": 107 | |
| }, | |
| { | |
| "clip_ratio": 0.0011543052969500422, | |
| "epoch": 0.0020148313977892823, | |
| "grad_norm": 0.07626564055681229, | |
| "kl": 0.02130126953125, | |
| "learning_rate": 4.603939897265268e-06, | |
| "loss": 0.0524, | |
| "step": 108 | |
| }, | |
| { | |
| "clip_ratio": 0.0008169686479959637, | |
| "completion_length": 897.7500305175781, | |
| "epoch": 0.002033487244065109, | |
| "grad_norm": 0.08059542626142502, | |
| "kl": 0.02227783203125, | |
| "learning_rate": 4.595642789309492e-06, | |
| "loss": 0.0107, | |
| "num_tokens": 1259393.0, | |
| "reward": 0.4133833199739456, | |
| "reward_std": 0.4549361765384674, | |
| "rewards/code_reward": 0.3917166441679001, | |
| "rewards/format_reward": 0.21666666865348816, | |
| "step": 109 | |
| }, | |
| { | |
| "clip_ratio": 0.0010060299537144601, | |
| "epoch": 0.0020521430903409354, | |
| "grad_norm": 0.08048728853464127, | |
| "kl": 0.0216064453125, | |
| "learning_rate": 4.587268241878724e-06, | |
| "loss": 0.0106, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.0012289943406358361, | |
| "epoch": 0.0020707989366167624, | |
| "grad_norm": 0.07647143304347992, | |
| "kl": 0.019775390625, | |
| "learning_rate": 4.578816606352205e-06, | |
| "loss": 0.0107, | |
| "step": 111 | |
| }, | |
| { | |
| "clip_ratio": 0.0013880816695746034, | |
| "completion_length": 1007.6667175292969, | |
| "epoch": 0.002089454782892589, | |
| "grad_norm": 0.07475873827934265, | |
| "kl": 0.01617431640625, | |
| "learning_rate": 4.570288237343632e-06, | |
| "loss": 0.0233, | |
| "num_tokens": 1298889.0, | |
| "reward": 0.16264833509922028, | |
| "reward_std": 0.22893914580345154, | |
| "rewards/code_reward": 0.14723165333271027, | |
| "rewards/format_reward": 0.15416666120290756, | |
| "step": 112 | |
| }, | |
| { | |
| "clip_ratio": 0.0013335386465769261, | |
| "epoch": 0.0021081106291684156, | |
| "grad_norm": 0.07221872359514236, | |
| "kl": 0.01495361328125, | |
| "learning_rate": 4.561683492686289e-06, | |
| "loss": 0.0231, | |
| "step": 113 | |
| }, | |
| { | |
| "clip_ratio": 0.0011128429905511439, | |
| "epoch": 0.0021267664754442426, | |
| "grad_norm": 0.11232966184616089, | |
| "kl": 0.013580322265625, | |
| "learning_rate": 4.5530027334180285e-06, | |
| "loss": 0.0232, | |
| "step": 114 | |
| }, | |
| { | |
| "clip_ratio": 0.0007180829998105764, | |
| "completion_length": 760.375, | |
| "epoch": 0.002145422321720069, | |
| "grad_norm": 0.08500614017248154, | |
| "kl": 0.0218505859375, | |
| "learning_rate": 4.544246323766122e-06, | |
| "loss": 0.0176, | |
| "num_tokens": 1329738.0, | |
| "reward": 0.42050473392009735, | |
| "reward_std": 0.4651963710784912, | |
| "rewards/code_reward": 0.37967143952846527, | |
| "rewards/format_reward": 0.40833334624767303, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_ratio": 0.0013084964302834123, | |
| "epoch": 0.0021640781679958957, | |
| "grad_norm": 0.09566804766654968, | |
| "kl": 0.01910400390625, | |
| "learning_rate": 4.535414631131983e-06, | |
| "loss": 0.0174, | |
| "step": 116 | |
| }, | |
| { | |
| "clip_ratio": 0.0012152747949585319, | |
| "epoch": 0.0021827340142717222, | |
| "grad_norm": 0.09403965622186661, | |
| "kl": 0.01824951171875, | |
| "learning_rate": 4.526508026075746e-06, | |
| "loss": 0.0175, | |
| "step": 117 | |
| }, | |
| { | |
| "clip_ratio": 0.00110370441689156, | |
| "completion_length": 913.9583740234375, | |
| "epoch": 0.0022013898605475492, | |
| "grad_norm": 0.09698626399040222, | |
| "kl": 0.022411346435546875, | |
| "learning_rate": 4.517526882300721e-06, | |
| "loss": -0.0229, | |
| "num_tokens": 1372757.0, | |
| "reward": 0.3174999728798866, | |
| "reward_std": 0.27699745260179043, | |
| "rewards/code_reward": 0.2916666567325592, | |
| "rewards/format_reward": 0.25833334028720856, | |
| "step": 118 | |
| }, | |
| { | |
| "clip_ratio": 0.0013115269830450416, | |
| "epoch": 0.002220045706823376, | |
| "grad_norm": 0.09875179827213287, | |
| "kl": 0.019748687744140625, | |
| "learning_rate": 4.508471576637713e-06, | |
| "loss": -0.0228, | |
| "step": 119 | |
| }, | |
| { | |
| "clip_ratio": 0.0011575823882594705, | |
| "epoch": 0.0022387015530992023, | |
| "grad_norm": 0.10077529400587082, | |
| "kl": 0.01830291748046875, | |
| "learning_rate": 4.499342489029211e-06, | |
| "loss": -0.0231, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.0006889864162076265, | |
| "completion_length": 498.3750305175781, | |
| "epoch": 0.0022573573993750293, | |
| "grad_norm": 0.11392726749181747, | |
| "kl": 0.025848388671875, | |
| "learning_rate": 4.490140002513449e-06, | |
| "loss": 0.027, | |
| "num_tokens": 1393730.0, | |
| "reward": 0.9718478322029114, | |
| "reward_std": 0.2110026255249977, | |
| "rewards/code_reward": 0.9293478429317474, | |
| "rewards/format_reward": 0.42499999701976776, | |
| "step": 121 | |
| }, | |
| { | |
| "clip_ratio": 0.0005561001598834991, | |
| "epoch": 0.002276013245650856, | |
| "grad_norm": 0.10952252149581909, | |
| "kl": 0.025726318359375, | |
| "learning_rate": 4.48086450320833e-06, | |
| "loss": 0.0267, | |
| "step": 122 | |
| }, | |
| { | |
| "clip_ratio": 0.0006027935305610299, | |
| "epoch": 0.0022946690919266825, | |
| "grad_norm": 0.1085001602768898, | |
| "kl": 0.0263671875, | |
| "learning_rate": 4.4715163802952266e-06, | |
| "loss": 0.0269, | |
| "step": 123 | |
| }, | |
| { | |
| "clip_ratio": 0.001191554736578837, | |
| "completion_length": 732.75, | |
| "epoch": 0.002313324938202509, | |
| "grad_norm": 0.08821776509284973, | |
| "kl": 0.031219482421875, | |
| "learning_rate": 4.462096026002655e-06, | |
| "loss": 0.0062, | |
| "num_tokens": 1422824.0, | |
| "reward": 0.03973765671253204, | |
| "reward_std": 0.03781243972480297, | |
| "rewards/code_reward": 0.00848765391856432, | |
| "rewards/format_reward": 0.3125000074505806, | |
| "step": 124 | |
| }, | |
| { | |
| "clip_ratio": 0.001063963572960347, | |
| "epoch": 0.002331980784478336, | |
| "grad_norm": 0.09551744163036346, | |
| "kl": 0.0391845703125, | |
| "learning_rate": 4.4526038355898144e-06, | |
| "loss": 0.0061, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.0010462789214216173, | |
| "epoch": 0.0023506366307541626, | |
| "grad_norm": 0.08740315586328506, | |
| "kl": 0.03314208984375, | |
| "learning_rate": 4.4430402073300035e-06, | |
| "loss": 0.0058, | |
| "step": 126 | |
| }, | |
| { | |
| "clip_ratio": 0.0009691159357316792, | |
| "completion_length": 1037.7500610351562, | |
| "epoch": 0.002369292477029989, | |
| "grad_norm": 0.07823047786951065, | |
| "kl": 0.0126953125, | |
| "learning_rate": 4.433405542493909e-06, | |
| "loss": -0.0015, | |
| "num_tokens": 1459130.0, | |
| "reward": 0.6776870489120483, | |
| "reward_std": 0.10291638784110546, | |
| "rewards/code_reward": 0.6343537420034409, | |
| "rewards/format_reward": 0.4333333447575569, | |
| "step": 127 | |
| }, | |
| { | |
| "clip_ratio": 0.001145587011706084, | |
| "epoch": 0.002387948323305816, | |
| "grad_norm": 0.07594677060842514, | |
| "kl": 0.012725830078125, | |
| "learning_rate": 4.4237002453327734e-06, | |
| "loss": -0.0017, | |
| "step": 128 | |
| }, | |
| { | |
| "clip_ratio": 0.0008910182514227927, | |
| "epoch": 0.0024066041695816427, | |
| "grad_norm": 0.07640928775072098, | |
| "kl": 0.0137939453125, | |
| "learning_rate": 4.4139247230614245e-06, | |
| "loss": -0.0019, | |
| "step": 129 | |
| }, | |
| { | |
| "clip_ratio": 0.0011884173727594316, | |
| "completion_length": 1120.7916870117188, | |
| "epoch": 0.0024252600158574692, | |
| "grad_norm": 0.07030566036701202, | |
| "kl": 0.014190673828125, | |
| "learning_rate": 4.404079385841201e-06, | |
| "loss": 0.0265, | |
| "num_tokens": 1506729.0, | |
| "reward": 0.7509399354457855, | |
| "reward_std": 0.3766524940729141, | |
| "rewards/code_reward": 0.7180232405662537, | |
| "rewards/format_reward": 0.3291666656732559, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.0011372754233889282, | |
| "epoch": 0.002443915862133296, | |
| "grad_norm": 0.07563882321119308, | |
| "kl": 0.015655517578125, | |
| "learning_rate": 4.394164646762734e-06, | |
| "loss": 0.0264, | |
| "step": 131 | |
| }, | |
| { | |
| "clip_ratio": 0.0009669371065683663, | |
| "epoch": 0.002462571708409123, | |
| "grad_norm": 0.07320787757635117, | |
| "kl": 0.014190673828125, | |
| "learning_rate": 4.384180921828618e-06, | |
| "loss": 0.0263, | |
| "step": 132 | |
| }, | |
| { | |
| "clip_ratio": 0.0012862688163295388, | |
| "completion_length": 733.4166870117188, | |
| "epoch": 0.0024812275546849494, | |
| "grad_norm": 0.09955161809921265, | |
| "kl": 0.013763427734375, | |
| "learning_rate": 4.374128629935955e-06, | |
| "loss": 0.0072, | |
| "num_tokens": 1537483.0, | |
| "reward": 0.5765306204557419, | |
| "reward_std": 0.14132879488170147, | |
| "rewards/code_reward": 0.5348639450967312, | |
| "rewards/format_reward": 0.4166666716337204, | |
| "step": 133 | |
| }, | |
| { | |
| "clip_ratio": 0.0008828279678709805, | |
| "epoch": 0.002499883400960776, | |
| "grad_norm": 0.10158013552427292, | |
| "kl": 0.014129638671875, | |
| "learning_rate": 4.364008192858781e-06, | |
| "loss": 0.0072, | |
| "step": 134 | |
| }, | |
| { | |
| "clip_ratio": 0.0012895396212115884, | |
| "epoch": 0.002518539247236603, | |
| "grad_norm": 0.08935025334358215, | |
| "kl": 0.01409912109375, | |
| "learning_rate": 4.353820035230366e-06, | |
| "loss": 0.007, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.000699867494404316, | |
| "completion_length": 921.0833435058594, | |
| "epoch": 0.0025371950935124295, | |
| "grad_norm": 0.07964594662189484, | |
| "kl": 0.00946044921875, | |
| "learning_rate": 4.3435645845254e-06, | |
| "loss": 0.0365, | |
| "num_tokens": 1577169.0, | |
| "reward": 0.04625000339001417, | |
| "reward_std": 0.0623414209112525, | |
| "rewards/code_reward": 0.02083333395421505, | |
| "rewards/format_reward": 0.2541666626930237, | |
| "step": 136 | |
| }, | |
| { | |
| "clip_ratio": 0.0003799215774051845, | |
| "epoch": 0.002555850939788256, | |
| "grad_norm": 0.08275993913412094, | |
| "kl": 0.010284423828125, | |
| "learning_rate": 4.333242271042054e-06, | |
| "loss": 0.0363, | |
| "step": 137 | |
| }, | |
| { | |
| "clip_ratio": 0.00039069188642315567, | |
| "epoch": 0.002574506786064083, | |
| "grad_norm": 0.08476851880550385, | |
| "kl": 0.0107574462890625, | |
| "learning_rate": 4.32285352788393e-06, | |
| "loss": 0.0363, | |
| "step": 138 | |
| }, | |
| { | |
| "clip_ratio": 0.0006457642884925008, | |
| "completion_length": 873.1250305175781, | |
| "epoch": 0.0025931626323399096, | |
| "grad_norm": 0.11129351705312729, | |
| "kl": 0.015472412109375, | |
| "learning_rate": 4.312398790941882e-06, | |
| "loss": 0.0132, | |
| "num_tokens": 1613100.0, | |
| "reward": 0.2220833506435156, | |
| "reward_std": 0.12337806262075901, | |
| "rewards/code_reward": 0.17291666567325592, | |
| "rewards/format_reward": 0.49166665971279144, | |
| "step": 139 | |
| }, | |
| { | |
| "clip_ratio": 0.0009015932446345687, | |
| "epoch": 0.002611818478615736, | |
| "grad_norm": 0.09786652773618698, | |
| "kl": 0.01611328125, | |
| "learning_rate": 4.301878498875735e-06, | |
| "loss": 0.0132, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.000771370017901063, | |
| "epoch": 0.0026304743248915627, | |
| "grad_norm": 0.09219111502170563, | |
| "kl": 0.017333984375, | |
| "learning_rate": 4.291293093095873e-06, | |
| "loss": 0.0128, | |
| "step": 141 | |
| }, | |
| { | |
| "clip_ratio": 0.00134377705398947, | |
| "completion_length": 758.8333740234375, | |
| "epoch": 0.0026491301711673897, | |
| "grad_norm": 0.09211676567792892, | |
| "kl": 0.048370361328125, | |
| "learning_rate": 4.280643017744723e-06, | |
| "loss": 0.0087, | |
| "num_tokens": 1640192.0, | |
| "reward": 0.4495016485452652, | |
| "reward_std": 0.13479312881827354, | |
| "rewards/code_reward": 0.3999183177947998, | |
| "rewards/format_reward": 0.4958333224058151, | |
| "step": 142 | |
| }, | |
| { | |
| "clip_ratio": 0.0010106293557328172, | |
| "epoch": 0.0026677860174432163, | |
| "grad_norm": 0.0978454202413559, | |
| "kl": 0.055938720703125, | |
| "learning_rate": 4.269928719678117e-06, | |
| "loss": 0.0087, | |
| "step": 143 | |
| }, | |
| { | |
| "clip_ratio": 0.0008834551408654079, | |
| "epoch": 0.002686441863719043, | |
| "grad_norm": 0.09563044458627701, | |
| "kl": 0.056365966796875, | |
| "learning_rate": 4.2591506484465426e-06, | |
| "loss": 0.0087, | |
| "step": 144 | |
| }, | |
| { | |
| "clip_ratio": 0.0010417406738270074, | |
| "completion_length": 811.8333740234375, | |
| "epoch": 0.00270509770999487, | |
| "grad_norm": 0.09586314111948013, | |
| "kl": 0.0595703125, | |
| "learning_rate": 4.248309256276283e-06, | |
| "loss": 0.0134, | |
| "num_tokens": 1674328.0, | |
| "reward": 0.4281481206417084, | |
| "reward_std": 0.351884625852108, | |
| "rewards/code_reward": 0.39814815670251846, | |
| "rewards/format_reward": 0.29999999701976776, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.0013275437522679567, | |
| "epoch": 0.0027237535562706964, | |
| "grad_norm": 0.08591298758983612, | |
| "kl": 0.050048828125, | |
| "learning_rate": 4.23740499805044e-06, | |
| "loss": 0.0134, | |
| "step": 146 | |
| }, | |
| { | |
| "clip_ratio": 0.0016137410420924425, | |
| "epoch": 0.002742409402546523, | |
| "grad_norm": 0.08595118671655655, | |
| "kl": 0.03662109375, | |
| "learning_rate": 4.22643833128985e-06, | |
| "loss": 0.0129, | |
| "step": 147 | |
| }, | |
| { | |
| "clip_ratio": 0.00041602051351219416, | |
| "completion_length": 819.7083740234375, | |
| "epoch": 0.0027610652488223495, | |
| "grad_norm": 0.09021999686956406, | |
| "kl": 0.03216552734375, | |
| "learning_rate": 4.215409716133885e-06, | |
| "loss": 0.0092, | |
| "num_tokens": 1707417.0, | |
| "reward": 0.42956283316016197, | |
| "reward_std": 0.21959564834833145, | |
| "rewards/code_reward": 0.38414610736072063, | |
| "rewards/format_reward": 0.4541666805744171, | |
| "step": 148 | |
| }, | |
| { | |
| "clip_ratio": 0.0007702489383518696, | |
| "epoch": 0.0027797210950981765, | |
| "grad_norm": 0.08686275035142899, | |
| "kl": 0.023193359375, | |
| "learning_rate": 4.204319615321151e-06, | |
| "loss": 0.0092, | |
| "step": 149 | |
| }, | |
| { | |
| "clip_ratio": 0.0006736835566698574, | |
| "epoch": 0.002798376941374003, | |
| "grad_norm": 0.08748073130846024, | |
| "kl": 0.01898193359375, | |
| "learning_rate": 4.193168494170065e-06, | |
| "loss": 0.009, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.0005513402575161308, | |
| "completion_length": 924.9166870117188, | |
| "epoch": 0.0028170327876498296, | |
| "grad_norm": 0.08234385401010513, | |
| "kl": 0.008636474609375, | |
| "learning_rate": 4.181956820559339e-06, | |
| "loss": -0.0217, | |
| "num_tokens": 1751527.0, | |
| "reward": 0.5591950714588165, | |
| "reward_std": 0.18445394933223724, | |
| "rewards/code_reward": 0.5187783911824226, | |
| "rewards/format_reward": 0.40416665375232697, | |
| "step": 151 | |
| }, | |
| { | |
| "clip_ratio": 0.0006712005124427378, | |
| "epoch": 0.0028356886339256566, | |
| "grad_norm": 0.0800977274775505, | |
| "kl": 0.006988525390625, | |
| "learning_rate": 4.170685064908342e-06, | |
| "loss": -0.0219, | |
| "step": 152 | |
| }, | |
| { | |
| "clip_ratio": 0.0005624309997074306, | |
| "epoch": 0.002854344480201483, | |
| "grad_norm": 0.08224301040172577, | |
| "kl": 0.0066986083984375, | |
| "learning_rate": 4.159353700157365e-06, | |
| "loss": -0.022, | |
| "step": 153 | |
| }, | |
| { | |
| "clip_ratio": 0.00046220331569202244, | |
| "completion_length": 557.3333435058594, | |
| "epoch": 0.0028730003264773097, | |
| "grad_norm": 0.09233321249485016, | |
| "kl": 0.028167724609375, | |
| "learning_rate": 4.14796320174778e-06, | |
| "loss": -0.013, | |
| "num_tokens": 1781007.0, | |
| "reward": 0.6308333575725555, | |
| "reward_std": 0.449993371963501, | |
| "rewards/code_reward": 0.5833333283662796, | |
| "rewards/format_reward": 0.4750000089406967, | |
| "step": 154 | |
| }, | |
| { | |
| "clip_ratio": 0.0002166997583117336, | |
| "epoch": 0.0028916561727531367, | |
| "grad_norm": 0.0909167230129242, | |
| "kl": 0.02734375, | |
| "learning_rate": 4.136514047602087e-06, | |
| "loss": -0.0133, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.00029903331596869975, | |
| "epoch": 0.0029103120190289633, | |
| "grad_norm": 0.09201790392398834, | |
| "kl": 0.0225372314453125, | |
| "learning_rate": 4.1250067181038635e-06, | |
| "loss": -0.0133, | |
| "step": 156 | |
| }, | |
| { | |
| "clip_ratio": 0.000859909545397386, | |
| "completion_length": 1166.7083740234375, | |
| "epoch": 0.00292896786530479, | |
| "grad_norm": 0.12645356357097626, | |
| "kl": 0.0063934326171875, | |
| "learning_rate": 4.113441696077608e-06, | |
| "loss": 0.0052, | |
| "num_tokens": 1827500.0, | |
| "reward": 0.04448845051229, | |
| "reward_std": 0.028481798246502876, | |
| "rewards/code_reward": 0.009488449431955814, | |
| "rewards/format_reward": 0.3499999940395355, | |
| "step": 157 | |
| }, | |
| { | |
| "clip_ratio": 0.0006146969681140035, | |
| "epoch": 0.0029476237115806164, | |
| "grad_norm": 0.09026872366666794, | |
| "kl": 0.00640869140625, | |
| "learning_rate": 4.101819466768484e-06, | |
| "loss": 0.0051, | |
| "step": 158 | |
| }, | |
| { | |
| "clip_ratio": 0.0006830913771409541, | |
| "epoch": 0.0029662795578564434, | |
| "grad_norm": 0.09458412230014801, | |
| "kl": 0.007568359375, | |
| "learning_rate": 4.0901405178219535e-06, | |
| "loss": 0.005, | |
| "step": 159 | |
| }, | |
| { | |
| "clip_ratio": 0.0009271332528442144, | |
| "completion_length": 902.75, | |
| "epoch": 0.00298493540413227, | |
| "grad_norm": 0.09097299724817276, | |
| "kl": 0.0082550048828125, | |
| "learning_rate": 4.078405339263326e-06, | |
| "loss": 0.0325, | |
| "num_tokens": 1863974.0, | |
| "reward": 0.3194986581802368, | |
| "reward_std": 0.23043666034936905, | |
| "rewards/code_reward": 0.28074865974485874, | |
| "rewards/format_reward": 0.38750001788139343, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.001312960172072053, | |
| "epoch": 0.0030035912504080965, | |
| "grad_norm": 0.08883295208215714, | |
| "kl": 0.00946044921875, | |
| "learning_rate": 4.06661442347719e-06, | |
| "loss": 0.0322, | |
| "step": 161 | |
| }, | |
| { | |
| "clip_ratio": 0.00119069468928501, | |
| "epoch": 0.0030222470966839235, | |
| "grad_norm": 0.08946386724710464, | |
| "kl": 0.010009765625, | |
| "learning_rate": 4.054768265186758e-06, | |
| "loss": 0.0322, | |
| "step": 162 | |
| }, | |
| { | |
| "clip_ratio": 0.0016622182447463274, | |
| "completion_length": 831.4583435058594, | |
| "epoch": 0.00304090294295975, | |
| "grad_norm": 0.07739204913377762, | |
| "kl": 0.0092010498046875, | |
| "learning_rate": 4.0428673614331036e-06, | |
| "loss": -0.0206, | |
| "num_tokens": 1901017.0, | |
| "reward": 0.39075271785259247, | |
| "reward_std": 0.4852019101381302, | |
| "rewards/code_reward": 0.36116939783096313, | |
| "rewards/format_reward": 0.2958333343267441, | |
| "step": 163 | |
| }, | |
| { | |
| "clip_ratio": 0.0012105648347642273, | |
| "epoch": 0.0030595587892355766, | |
| "grad_norm": 0.07764855027198792, | |
| "kl": 0.00982666015625, | |
| "learning_rate": 4.030912211554316e-06, | |
| "loss": -0.0211, | |
| "step": 164 | |
| }, | |
| { | |
| "clip_ratio": 0.001361995586194098, | |
| "epoch": 0.003078214635511403, | |
| "grad_norm": 0.0787842720746994, | |
| "kl": 0.0101318359375, | |
| "learning_rate": 4.018903317164539e-06, | |
| "loss": -0.0208, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.00047827488742768764, | |
| "completion_length": 970.7916870117188, | |
| "epoch": 0.00309687048178723, | |
| "grad_norm": 0.09784507006406784, | |
| "kl": 0.010040283203125, | |
| "learning_rate": 4.006841182132932e-06, | |
| "loss": 0.0032, | |
| "num_tokens": 1933292.0, | |
| "reward": 0.7487499713897705, | |
| "reward_std": 0.4054149240255356, | |
| "rewards/code_reward": 0.7083333432674408, | |
| "rewards/format_reward": 0.40416666865348816, | |
| "step": 166 | |
| }, | |
| { | |
| "clip_ratio": 0.0005554754170589149, | |
| "epoch": 0.0031155263280630567, | |
| "grad_norm": 0.09923358261585236, | |
| "kl": 0.01019287109375, | |
| "learning_rate": 3.9947263125625195e-06, | |
| "loss": 0.0032, | |
| "step": 167 | |
| }, | |
| { | |
| "clip_ratio": 0.00030202302150428295, | |
| "epoch": 0.0031341821743388833, | |
| "grad_norm": 0.09393054246902466, | |
| "kl": 0.0109100341796875, | |
| "learning_rate": 3.982559216768967e-06, | |
| "loss": 0.0029, | |
| "step": 168 | |
| }, | |
| { | |
| "clip_ratio": 0.0007342632743529975, | |
| "completion_length": 645.7500305175781, | |
| "epoch": 0.0031528380206147103, | |
| "grad_norm": 0.09612008929252625, | |
| "kl": 0.01934814453125, | |
| "learning_rate": 3.970340405259245e-06, | |
| "loss": -0.0236, | |
| "num_tokens": 1956074.0, | |
| "reward": 0.6863030195236206, | |
| "reward_std": 0.3077765703201294, | |
| "rewards/code_reward": 0.6533863097429276, | |
| "rewards/format_reward": 0.3291666731238365, | |
| "step": 169 | |
| }, | |
| { | |
| "clip_ratio": 0.0007756684790365398, | |
| "epoch": 0.003171493866890537, | |
| "grad_norm": 0.0964791551232338, | |
| "kl": 0.02117919921875, | |
| "learning_rate": 3.958070390710214e-06, | |
| "loss": -0.0234, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.0010250475606881082, | |
| "epoch": 0.0031901497131663634, | |
| "grad_norm": 0.0937555730342865, | |
| "kl": 0.02099609375, | |
| "learning_rate": 3.945749687947109e-06, | |
| "loss": -0.0239, | |
| "step": 171 | |
| }, | |
| { | |
| "clip_ratio": 0.001571565808262676, | |
| "completion_length": 856.1666870117188, | |
| "epoch": 0.0032088055594421904, | |
| "grad_norm": 0.09850820899009705, | |
| "kl": 0.014801025390625, | |
| "learning_rate": 3.933378813921942e-06, | |
| "loss": 0.0194, | |
| "num_tokens": 1987470.0, | |
| "reward": 0.048796952702105045, | |
| "reward_std": 0.04604018107056618, | |
| "rewards/code_reward": 0.021713614463806152, | |
| "rewards/format_reward": 0.2708333358168602, | |
| "step": 172 | |
| }, | |
| { | |
| "clip_ratio": 0.0017044495907612145, | |
| "epoch": 0.003227461405718017, | |
| "grad_norm": 0.10293813049793243, | |
| "kl": 0.01483154296875, | |
| "learning_rate": 3.920958287691811e-06, | |
| "loss": 0.0193, | |
| "step": 173 | |
| }, | |
| { | |
| "clip_ratio": 0.0016478158067911863, | |
| "epoch": 0.0032461172519938435, | |
| "grad_norm": 0.09644638746976852, | |
| "kl": 0.017425537109375, | |
| "learning_rate": 3.908488630397121e-06, | |
| "loss": 0.0189, | |
| "step": 174 | |
| }, | |
| { | |
| "clip_ratio": 0.0009133272105827928, | |
| "completion_length": 797.375, | |
| "epoch": 0.00326477309826967, | |
| "grad_norm": 0.08966498076915741, | |
| "kl": 0.018890380859375, | |
| "learning_rate": 3.8959703652397175e-06, | |
| "loss": 0.0182, | |
| "num_tokens": 2022255.0, | |
| "reward": 0.4206944704055786, | |
| "reward_std": 0.16624127328395844, | |
| "rewards/code_reward": 0.3861111178994179, | |
| "rewards/format_reward": 0.34583333134651184, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_ratio": 0.0008382981468457729, | |
| "epoch": 0.003283428944545497, | |
| "grad_norm": 0.09205059707164764, | |
| "kl": 0.019439697265625, | |
| "learning_rate": 3.883404017460935e-06, | |
| "loss": 0.0181, | |
| "step": 176 | |
| }, | |
| { | |
| "clip_ratio": 0.000650079753540922, | |
| "epoch": 0.0033020847908213236, | |
| "grad_norm": 0.10440210998058319, | |
| "kl": 0.0201416015625, | |
| "learning_rate": 3.870790114319559e-06, | |
| "loss": 0.0182, | |
| "step": 177 | |
| }, | |
| { | |
| "clip_ratio": 0.00037280093238223344, | |
| "completion_length": 798.9166870117188, | |
| "epoch": 0.00332074063709715, | |
| "grad_norm": 0.09877095371484756, | |
| "kl": 0.0088043212890625, | |
| "learning_rate": 3.858129185069701e-06, | |
| "loss": 0.001, | |
| "num_tokens": 2058181.0, | |
| "reward": 0.6779166460037231, | |
| "reward_std": 0.4303756207227707, | |
| "rewards/code_reward": 0.625, | |
| "rewards/format_reward": 0.5291666686534882, | |
| "step": 178 | |
| }, | |
| { | |
| "clip_ratio": 0.0005341586947906762, | |
| "epoch": 0.003339396483372977, | |
| "grad_norm": 0.09430207312107086, | |
| "kl": 0.0088958740234375, | |
| "learning_rate": 3.845421760938597e-06, | |
| "loss": 0.0011, | |
| "step": 179 | |
| }, | |
| { | |
| "clip_ratio": 0.00040165129757951945, | |
| "epoch": 0.0033580523296488037, | |
| "grad_norm": 0.08880569040775299, | |
| "kl": 0.0091552734375, | |
| "learning_rate": 3.832668375104312e-06, | |
| "loss": 0.0008, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_ratio": 0.0010204909485764802, | |
| "completion_length": 789.0833435058594, | |
| "epoch": 0.0033767081759246303, | |
| "grad_norm": 0.0856408104300499, | |
| "kl": 0.0240478515625, | |
| "learning_rate": 3.8198695626733725e-06, | |
| "loss": 0.0168, | |
| "num_tokens": 2085147.0, | |
| "reward": 0.0904554259032011, | |
| "reward_std": 0.16545533016324043, | |
| "rewards/code_reward": 0.050872091203927994, | |
| "rewards/format_reward": 0.3958333283662796, | |
| "step": 181 | |
| }, | |
| { | |
| "clip_ratio": 0.0013127815909683704, | |
| "epoch": 0.003395364022200457, | |
| "grad_norm": 0.08718743920326233, | |
| "kl": 0.02423095703125, | |
| "learning_rate": 3.8070258606583156e-06, | |
| "loss": 0.0167, | |
| "step": 182 | |
| }, | |
| { | |
| "clip_ratio": 0.0010590656311251223, | |
| "epoch": 0.003414019868476284, | |
| "grad_norm": 0.08608557283878326, | |
| "kl": 0.02398681640625, | |
| "learning_rate": 3.7941378079551544e-06, | |
| "loss": 0.0163, | |
| "step": 183 | |
| }, | |
| { | |
| "clip_ratio": 0.00044808804523199797, | |
| "completion_length": 838.3333435058594, | |
| "epoch": 0.0034326757147521104, | |
| "grad_norm": 0.08882498741149902, | |
| "kl": 0.04205322265625, | |
| "learning_rate": 3.7812059453207677e-06, | |
| "loss": -0.0138, | |
| "num_tokens": 2118659.0, | |
| "reward": 0.5362499505281448, | |
| "reward_std": 0.020687240175902843, | |
| "rewards/code_reward": 0.5, | |
| "rewards/format_reward": 0.36249999701976776, | |
| "step": 184 | |
| }, | |
| { | |
| "clip_ratio": 0.0009204872185364366, | |
| "epoch": 0.003451331561027937, | |
| "grad_norm": 0.08916648477315903, | |
| "kl": 0.044921875, | |
| "learning_rate": 3.768230815350213e-06, | |
| "loss": -0.0138, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_ratio": 0.0004569297598209232, | |
| "epoch": 0.003469987407303764, | |
| "grad_norm": 0.08778577297925949, | |
| "kl": 0.0452880859375, | |
| "learning_rate": 3.7552129624539557e-06, | |
| "loss": -0.0139, | |
| "step": 186 | |
| }, | |
| { | |
| "clip_ratio": 0.0006422713631764054, | |
| "completion_length": 820.6666870117188, | |
| "epoch": 0.0034886432535795905, | |
| "grad_norm": 0.07521483302116394, | |
| "kl": 0.02374267578125, | |
| "learning_rate": 3.7421529328350316e-06, | |
| "loss": -0.0011, | |
| "num_tokens": 2154951.0, | |
| "reward": 0.3243994116783142, | |
| "reward_std": 0.16106001287698746, | |
| "rewards/code_reward": 0.26398275047540665, | |
| "rewards/format_reward": 0.6041666567325592, | |
| "step": 187 | |
| }, | |
| { | |
| "clip_ratio": 0.0007444258662872016, | |
| "epoch": 0.003507299099855417, | |
| "grad_norm": 0.07639120519161224, | |
| "kl": 0.022552490234375, | |
| "learning_rate": 3.7290512744661274e-06, | |
| "loss": -0.0014, | |
| "step": 188 | |
| }, | |
| { | |
| "clip_ratio": 0.0008128004556056112, | |
| "epoch": 0.003525954946131244, | |
| "grad_norm": 0.07584098726511002, | |
| "kl": 0.02203369140625, | |
| "learning_rate": 3.715908537066589e-06, | |
| "loss": -0.0016, | |
| "step": 189 | |
| }, | |
| { | |
| "clip_ratio": 0.0006247111014090478, | |
| "completion_length": 782.2500610351562, | |
| "epoch": 0.0035446107924070706, | |
| "grad_norm": 0.09555865079164505, | |
| "kl": 0.02532958984375, | |
| "learning_rate": 3.7027252720793538e-06, | |
| "loss": -0.0372, | |
| "num_tokens": 2188473.0, | |
| "reward": 0.640339195728302, | |
| "reward_std": 0.48519615828990936, | |
| "rewards/code_reward": 0.5903392136096954, | |
| "rewards/format_reward": 0.5, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_ratio": 0.0004575684142764658, | |
| "epoch": 0.003563266638682897, | |
| "grad_norm": 0.09690264612436295, | |
| "kl": 0.0252685546875, | |
| "learning_rate": 3.689502032647817e-06, | |
| "loss": -0.0374, | |
| "step": 191 | |
| }, | |
| { | |
| "clip_ratio": 0.0003101055626757443, | |
| "epoch": 0.0035819224849587238, | |
| "grad_norm": 0.08903127908706665, | |
| "kl": 0.02032470703125, | |
| "learning_rate": 3.6762393735926245e-06, | |
| "loss": -0.0377, | |
| "step": 192 | |
| }, | |
| { | |
| "clip_ratio": 0.00029177512624301016, | |
| "completion_length": 791.9583435058594, | |
| "epoch": 0.0036005783312345508, | |
| "grad_norm": 0.10319360345602036, | |
| "kl": 0.0065460205078125, | |
| "learning_rate": 3.6629378513883852e-06, | |
| "loss": 0.0565, | |
| "num_tokens": 2221952.0, | |
| "reward": 0.5221759006381035, | |
| "reward_std": 0.40614573657512665, | |
| "rewards/code_reward": 0.4759259298443794, | |
| "rewards/format_reward": 0.4624999910593033, | |
| "step": 193 | |
| }, | |
| { | |
| "clip_ratio": 0.0005471540498547256, | |
| "epoch": 0.0036192341775103773, | |
| "grad_norm": 0.07854603230953217, | |
| "kl": 0.00677490234375, | |
| "learning_rate": 3.6495980241403307e-06, | |
| "loss": 0.0563, | |
| "step": 194 | |
| }, | |
| { | |
| "clip_ratio": 0.0007658963440917432, | |
| "epoch": 0.003637890023786204, | |
| "grad_norm": 0.1044902577996254, | |
| "kl": 0.0064697265625, | |
| "learning_rate": 3.636220451560896e-06, | |
| "loss": 0.0565, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_ratio": 0.001431856129784137, | |
| "completion_length": 845.7916870117188, | |
| "epoch": 0.003656545870062031, | |
| "grad_norm": 0.08788370341062546, | |
| "kl": 0.015106201171875, | |
| "learning_rate": 3.622805694946235e-06, | |
| "loss": 0.0278, | |
| "num_tokens": 2256867.0, | |
| "reward": 0.55375000461936, | |
| "reward_std": 0.04919072613120079, | |
| "rewards/code_reward": 0.5, | |
| "rewards/format_reward": 0.5374999940395355, | |
| "step": 196 | |
| }, | |
| { | |
| "clip_ratio": 0.0012035187100991607, | |
| "epoch": 0.0036752017163378574, | |
| "grad_norm": 0.08841948956251144, | |
| "kl": 0.014556884765625, | |
| "learning_rate": 3.609354317152667e-06, | |
| "loss": 0.028, | |
| "step": 197 | |
| }, | |
| { | |
| "clip_ratio": 0.0011701789626386017, | |
| "epoch": 0.003693857562613684, | |
| "grad_norm": 0.08651836961507797, | |
| "kl": 0.015625, | |
| "learning_rate": 3.595866882573063e-06, | |
| "loss": 0.0275, | |
| "step": 198 | |
| }, | |
| { | |
| "clip_ratio": 0.0006608768162550405, | |
| "completion_length": 812.9583435058594, | |
| "epoch": 0.0037125134088895105, | |
| "grad_norm": 0.09646130353212357, | |
| "kl": 0.010162353515625, | |
| "learning_rate": 3.5823439571131675e-06, | |
| "loss": -0.0052, | |
| "num_tokens": 2286290.0, | |
| "reward": 0.09254807978868484, | |
| "reward_std": 0.06599834933876991, | |
| "rewards/code_reward": 0.030048077926039696, | |
| "rewards/format_reward": 0.625, | |
| "step": 199 | |
| }, | |
| { | |
| "clip_ratio": 0.000630842725513503, | |
| "epoch": 0.0037311692551653375, | |
| "grad_norm": 0.09436467289924622, | |
| "kl": 0.0110015869140625, | |
| "learning_rate": 3.5687861081678477e-06, | |
| "loss": -0.0052, | |
| "step": 200 | |
| }, | |
| { | |
| "clip_ratio": 0.0007917208422441036, | |
| "epoch": 0.003749825101441164, | |
| "grad_norm": 0.09504929929971695, | |
| "kl": 0.012542724609375, | |
| "learning_rate": 3.555193904597291e-06, | |
| "loss": -0.0053, | |
| "step": 201 | |
| }, | |
| { | |
| "clip_ratio": 0.0005567904736381024, | |
| "completion_length": 846.1250305175781, | |
| "epoch": 0.0037684809477169907, | |
| "grad_norm": 0.08655253797769547, | |
| "kl": 0.0111083984375, | |
| "learning_rate": 3.541567916703138e-06, | |
| "loss": 0.0168, | |
| "num_tokens": 2316833.0, | |
| "reward": 0.46897655725479126, | |
| "reward_std": 0.23874285072088242, | |
| "rewards/code_reward": 0.4368932172656059, | |
| "rewards/format_reward": 0.32083334028720856, | |
| "step": 202 | |
| }, | |
| { | |
| "clip_ratio": 0.0003927100624423474, | |
| "epoch": 0.0037871367939928176, | |
| "grad_norm": 0.08548427373170853, | |
| "kl": 0.01123046875, | |
| "learning_rate": 3.5279087162045517e-06, | |
| "loss": 0.0168, | |
| "step": 203 | |
| }, | |
| { | |
| "clip_ratio": 0.0006460061413235962, | |
| "epoch": 0.003805792640268644, | |
| "grad_norm": 0.08546783775091171, | |
| "kl": 0.011199951171875, | |
| "learning_rate": 3.5142168762142265e-06, | |
| "loss": 0.0165, | |
| "step": 204 | |
| }, | |
| { | |
| "clip_ratio": 0.0005783264059573412, | |
| "completion_length": 671.1666870117188, | |
| "epoch": 0.0038244484865444708, | |
| "grad_norm": 0.2734091579914093, | |
| "kl": 0.020843505859375, | |
| "learning_rate": 3.500492971214347e-06, | |
| "loss": 0.008, | |
| "num_tokens": 2344797.0, | |
| "reward": 0.5388242453336716, | |
| "reward_std": 0.3791409581899643, | |
| "rewards/code_reward": 0.4925742968916893, | |
| "rewards/format_reward": 0.4625000059604645, | |
| "step": 205 | |
| }, | |
| { | |
| "clip_ratio": 0.0003238898643758148, | |
| "epoch": 0.0038431043328202978, | |
| "grad_norm": 0.08638158440589905, | |
| "kl": 0.021759033203125, | |
| "learning_rate": 3.48673757703248e-06, | |
| "loss": 0.0078, | |
| "step": 206 | |
| }, | |
| { | |
| "clip_ratio": 0.00025367678608745337, | |
| "epoch": 0.0038617601790961243, | |
| "grad_norm": 0.08271009474992752, | |
| "kl": 0.02239990234375, | |
| "learning_rate": 3.472951270817418e-06, | |
| "loss": 0.0077, | |
| "step": 207 | |
| }, | |
| { | |
| "clip_ratio": 0.0011410006263758987, | |
| "completion_length": 863.6250610351562, | |
| "epoch": 0.003880416025371951, | |
| "grad_norm": 0.0871393084526062, | |
| "kl": 0.009246826171875, | |
| "learning_rate": 3.4591346310149578e-06, | |
| "loss": 0.0031, | |
| "num_tokens": 2380560.0, | |
| "reward": 0.07268582284450531, | |
| "reward_std": 0.057141270488500595, | |
| "rewards/code_reward": 0.028519157320261, | |
| "rewards/format_reward": 0.4416666626930237, | |
| "step": 208 | |
| }, | |
| { | |
| "clip_ratio": 0.0012201684294268489, | |
| "epoch": 0.0038990718716477774, | |
| "grad_norm": 0.08084117621183395, | |
| "kl": 0.00958251953125, | |
| "learning_rate": 3.445288237343632e-06, | |
| "loss": 0.0031, | |
| "step": 209 | |
| }, | |
| { | |
| "clip_ratio": 0.0008913867350202054, | |
| "epoch": 0.0039177277179236044, | |
| "grad_norm": 0.08293391019105911, | |
| "kl": 0.009307861328125, | |
| "learning_rate": 3.4314126707703895e-06, | |
| "loss": 0.003, | |
| "step": 210 | |
| }, | |
| { | |
| "clip_ratio": 0.0013628143933601677, | |
| "completion_length": 695.0000305175781, | |
| "epoch": 0.003936383564199431, | |
| "grad_norm": 0.10922668874263763, | |
| "kl": 0.0301513671875, | |
| "learning_rate": 3.4175085134862128e-06, | |
| "loss": 0.0343, | |
| "num_tokens": 2409312.0, | |
| "reward": 0.2725326791405678, | |
| "reward_std": 0.31870152056217194, | |
| "rewards/code_reward": 0.24836601875722408, | |
| "rewards/format_reward": 0.24166666716337204, | |
| "step": 211 | |
| }, | |
| { | |
| "clip_ratio": 0.0010863369097933173, | |
| "epoch": 0.0039550394104752576, | |
| "grad_norm": 0.09670348465442657, | |
| "kl": 0.024658203125, | |
| "learning_rate": 3.4035763488816953e-06, | |
| "loss": 0.0341, | |
| "step": 212 | |
| }, | |
| { | |
| "clip_ratio": 0.0009245816036127508, | |
| "epoch": 0.003973695256751084, | |
| "grad_norm": 0.0888768658041954, | |
| "kl": 0.02496337890625, | |
| "learning_rate": 3.3896167615225594e-06, | |
| "loss": 0.0339, | |
| "step": 213 | |
| }, | |
| { | |
| "clip_ratio": 0.0007956430781632662, | |
| "completion_length": 803.4583435058594, | |
| "epoch": 0.003992351103026911, | |
| "grad_norm": 0.08351018279790878, | |
| "kl": 0.010467529296875, | |
| "learning_rate": 3.375630337125133e-06, | |
| "loss": -0.0005, | |
| "num_tokens": 2439047.0, | |
| "reward": 0.2963353507220745, | |
| "reward_std": 0.2816299609839916, | |
| "rewards/code_reward": 0.25050200801342726, | |
| "rewards/format_reward": 0.4583333432674408, | |
| "step": 214 | |
| }, | |
| { | |
| "clip_ratio": 0.0009151631966233253, | |
| "epoch": 0.004011006949302738, | |
| "grad_norm": 0.0963514894247055, | |
| "kl": 0.010101318359375, | |
| "learning_rate": 3.361617662531772e-06, | |
| "loss": -0.0006, | |
| "step": 215 | |
| }, | |
| { | |
| "clip_ratio": 0.0005119523120811209, | |
| "epoch": 0.004029662795578565, | |
| "grad_norm": 0.08659033477306366, | |
| "kl": 0.0107421875, | |
| "learning_rate": 3.347579325686237e-06, | |
| "loss": -0.0011, | |
| "step": 216 | |
| }, | |
| { | |
| "clip_ratio": 0.0005694587889593095, | |
| "completion_length": 548.8750305175781, | |
| "epoch": 0.004048318641854391, | |
| "grad_norm": 0.09764964878559113, | |
| "kl": 0.015594482421875, | |
| "learning_rate": 3.333515915609027e-06, | |
| "loss": 0.0127, | |
| "num_tokens": 2463224.0, | |
| "reward": 0.8187499940395355, | |
| "reward_std": 0.4159911423921585, | |
| "rewards/code_reward": 0.75, | |
| "rewards/format_reward": 0.6875, | |
| "step": 217 | |
| }, | |
| { | |
| "clip_ratio": 0.0006374841468641534, | |
| "epoch": 0.004066974488130218, | |
| "grad_norm": 0.09726841002702713, | |
| "kl": 0.01580810546875, | |
| "learning_rate": 3.3194280223726616e-06, | |
| "loss": 0.0127, | |
| "step": 218 | |
| }, | |
| { | |
| "clip_ratio": 0.0008805101679172367, | |
| "epoch": 0.004085630334406044, | |
| "grad_norm": 0.09693842381238937, | |
| "kl": 0.016326904296875, | |
| "learning_rate": 3.305316237076927e-06, | |
| "loss": 0.0126, | |
| "step": 219 | |
| }, | |
| { | |
| "clip_ratio": 0.001065359654603526, | |
| "completion_length": 682.2083435058594, | |
| "epoch": 0.004104286180681871, | |
| "grad_norm": 0.11235346645116806, | |
| "kl": 0.00555419921875, | |
| "learning_rate": 3.291181151824071e-06, | |
| "loss": 0.0219, | |
| "num_tokens": 2487829.0, | |
| "reward": 0.851570725440979, | |
| "reward_std": 0.13773459196090698, | |
| "rewards/code_reward": 0.7932373583316803, | |
| "rewards/format_reward": 0.5833333134651184, | |
| "step": 220 | |
| }, | |
| { | |
| "clip_ratio": 0.0010035349405370653, | |
| "epoch": 0.0041229420269576975, | |
| "grad_norm": 0.10121559351682663, | |
| "kl": 0.0056304931640625, | |
| "learning_rate": 3.27702335969396e-06, | |
| "loss": 0.0218, | |
| "step": 221 | |
| }, | |
| { | |
| "clip_ratio": 0.0007600956014357507, | |
| "epoch": 0.004141597873233525, | |
| "grad_norm": 0.09749292582273483, | |
| "kl": 0.0055694580078125, | |
| "learning_rate": 3.2628434547191985e-06, | |
| "loss": 0.0217, | |
| "step": 222 | |
| }, | |
| { | |
| "clip_ratio": 0.00039983812894206494, | |
| "completion_length": 822.5416870117188, | |
| "epoch": 0.0041602537195093514, | |
| "grad_norm": 0.08266626298427582, | |
| "kl": 0.0115966796875, | |
| "learning_rate": 3.2486420318601973e-06, | |
| "loss": -0.0072, | |
| "num_tokens": 2524514.0, | |
| "reward": 0.765346884727478, | |
| "reward_std": 0.41189485788345337, | |
| "rewards/code_reward": 0.6945134997367859, | |
| "rewards/format_reward": 0.7083333283662796, | |
| "step": 223 | |
| }, | |
| { | |
| "clip_ratio": 0.00040171146974898875, | |
| "epoch": 0.004178909565785178, | |
| "grad_norm": 0.08238179981708527, | |
| "kl": 0.0108184814453125, | |
| "learning_rate": 3.2344196869802187e-06, | |
| "loss": -0.0073, | |
| "step": 224 | |
| }, | |
| { | |
| "clip_ratio": 0.0007332674576900899, | |
| "epoch": 0.0041975654120610046, | |
| "grad_norm": 0.08556170016527176, | |
| "kl": 0.0115203857421875, | |
| "learning_rate": 3.2201770168203694e-06, | |
| "loss": -0.0072, | |
| "step": 225 | |
| }, | |
| { | |
| "clip_ratio": 0.0010465582308825105, | |
| "completion_length": 897.4583435058594, | |
| "epoch": 0.004216221258336831, | |
| "grad_norm": 0.0795939639210701, | |
| "kl": 0.010223388671875, | |
| "learning_rate": 3.205914618974563e-06, | |
| "loss": 0.0444, | |
| "num_tokens": 2561713.0, | |
| "reward": 0.5436052978038788, | |
| "reward_std": 0.36193977296352386, | |
| "rewards/code_reward": 0.49235527217388153, | |
| "rewards/format_reward": 0.5125000178813934, | |
| "step": 226 | |
| }, | |
| { | |
| "clip_ratio": 0.001006987877190113, | |
| "epoch": 0.004234877104612658, | |
| "grad_norm": 0.0830455869436264, | |
| "kl": 0.00970458984375, | |
| "learning_rate": 3.1916330918644496e-06, | |
| "loss": 0.0443, | |
| "step": 227 | |
| }, | |
| { | |
| "clip_ratio": 0.0006825106975156814, | |
| "epoch": 0.004253532950888485, | |
| "grad_norm": 0.07931126654148102, | |
| "kl": 0.009521484375, | |
| "learning_rate": 3.177333034714303e-06, | |
| "loss": 0.0445, | |
| "step": 228 | |
| }, | |
| { | |
| "clip_ratio": 0.0010472153662703931, | |
| "completion_length": 1060.8333740234375, | |
| "epoch": 0.004272188797164312, | |
| "grad_norm": 0.08189205825328827, | |
| "kl": 0.01904296875, | |
| "learning_rate": 3.1630150475258813e-06, | |
| "loss": 0.0354, | |
| "num_tokens": 2602557.0, | |
| "reward": 0.5575000084936619, | |
| "reward_std": 0.09106174670159817, | |
| "rewards/code_reward": 0.520833333954215, | |
| "rewards/format_reward": 0.36666667461395264, | |
| "step": 229 | |
| }, | |
| { | |
| "clip_ratio": 0.0012223481899127364, | |
| "epoch": 0.004290844643440138, | |
| "grad_norm": 0.07629529386758804, | |
| "kl": 0.01898193359375, | |
| "learning_rate": 3.148679731053252e-06, | |
| "loss": 0.0355, | |
| "step": 230 | |
| }, | |
| { | |
| "clip_ratio": 0.0009530899405945092, | |
| "epoch": 0.004309500489715965, | |
| "grad_norm": 0.07555654644966125, | |
| "kl": 0.01959228515625, | |
| "learning_rate": 3.1343276867775805e-06, | |
| "loss": 0.0352, | |
| "step": 231 | |
| }, | |
| { | |
| "clip_ratio": 0.000721571734175086, | |
| "completion_length": 631.0, | |
| "epoch": 0.004328156335991791, | |
| "grad_norm": 0.08854309469461441, | |
| "kl": 0.01287841796875, | |
| "learning_rate": 3.1199595168819043e-06, | |
| "loss": -0.014, | |
| "num_tokens": 2627097.0, | |
| "reward": 0.7598794400691986, | |
| "reward_std": 0.4630916863679886, | |
| "rewards/code_reward": 0.6998794674873352, | |
| "rewards/format_reward": 0.6000000238418579, | |
| "step": 232 | |
| }, | |
| { | |
| "clip_ratio": 0.0004376633296487853, | |
| "epoch": 0.004346812182267618, | |
| "grad_norm": 0.1049780622124672, | |
| "kl": 0.01348876953125, | |
| "learning_rate": 3.105575824225852e-06, | |
| "loss": -0.0143, | |
| "step": 233 | |
| }, | |
| { | |
| "clip_ratio": 0.00047352669935207814, | |
| "epoch": 0.0043654680285434445, | |
| "grad_norm": 0.08706912398338318, | |
| "kl": 0.01336669921875, | |
| "learning_rate": 3.091177212320363e-06, | |
| "loss": -0.0145, | |
| "step": 234 | |
| }, | |
| { | |
| "clip_ratio": 0.00033213794813491404, | |
| "completion_length": 1132.4166870117188, | |
| "epoch": 0.004384123874819272, | |
| "grad_norm": 0.08825133740901947, | |
| "kl": 0.01216888427734375, | |
| "learning_rate": 3.0767642853023538e-06, | |
| "loss": -0.0049, | |
| "num_tokens": 2672143.0, | |
| "reward": 0.09307292103767395, | |
| "reward_std": 0.049834271892905235, | |
| "rewards/code_reward": 0.039322917349636555, | |
| "rewards/format_reward": 0.5375000089406967, | |
| "step": 235 | |
| }, | |
| { | |
| "clip_ratio": 0.0003840103163383901, | |
| "epoch": 0.0044027797210950985, | |
| "grad_norm": 0.09255920350551605, | |
| "kl": 0.01175689697265625, | |
| "learning_rate": 3.062337647909376e-06, | |
| "loss": -0.0051, | |
| "step": 236 | |
| }, | |
| { | |
| "clip_ratio": 0.00038397236494347453, | |
| "epoch": 0.004421435567370925, | |
| "grad_norm": 0.08827126771211624, | |
| "kl": 0.010589599609375, | |
| "learning_rate": 3.04789790545424e-06, | |
| "loss": -0.0052, | |
| "step": 237 | |
| }, | |
| { | |
| "clip_ratio": 0.0007554197800345719, | |
| "completion_length": 795.25, | |
| "epoch": 0.004440091413646752, | |
| "grad_norm": 0.09881194680929184, | |
| "kl": 0.02008056640625, | |
| "learning_rate": 3.033445663799621e-06, | |
| "loss": 0.0047, | |
| "num_tokens": 2704009.0, | |
| "reward": 0.5524518974125385, | |
| "reward_std": 0.056896304711699486, | |
| "rewards/code_reward": 0.5012019231216982, | |
| "rewards/format_reward": 0.5125000029802322, | |
| "step": 238 | |
| }, | |
| { | |
| "clip_ratio": 0.0010150580055778846, | |
| "epoch": 0.004458747259922578, | |
| "grad_norm": 0.09783502668142319, | |
| "kl": 0.021484375, | |
| "learning_rate": 3.018981529332633e-06, | |
| "loss": 0.0046, | |
| "step": 239 | |
| }, | |
| { | |
| "clip_ratio": 0.0011725955409929156, | |
| "epoch": 0.004477403106198405, | |
| "grad_norm": 0.09466077387332916, | |
| "kl": 0.0211181640625, | |
| "learning_rate": 3.00450610893939e-06, | |
| "loss": 0.0045, | |
| "step": 240 | |
| }, | |
| { | |
| "clip_ratio": 0.0011200214212294668, | |
| "completion_length": 709.3750152587891, | |
| "epoch": 0.004496058952474231, | |
| "grad_norm": 0.11234831809997559, | |
| "kl": 0.014251708984375, | |
| "learning_rate": 2.9900200099795396e-06, | |
| "loss": -0.001, | |
| "num_tokens": 2735914.0, | |
| "reward": 0.33983870036900043, | |
| "reward_std": 0.2842044336721301, | |
| "rewards/code_reward": 0.29233869910240173, | |
| "rewards/format_reward": 0.4750000089406967, | |
| "step": 241 | |
| }, | |
| { | |
| "clip_ratio": 0.0012173219583928585, | |
| "epoch": 0.004514714798750059, | |
| "grad_norm": 0.10929913818836212, | |
| "kl": 0.015838623046875, | |
| "learning_rate": 2.9755238402607826e-06, | |
| "loss": -0.0009, | |
| "step": 242 | |
| }, | |
| { | |
| "clip_ratio": 0.0010618427768349648, | |
| "epoch": 0.004533370645025885, | |
| "grad_norm": 0.1097099706530571, | |
| "kl": 0.017913818359375, | |
| "learning_rate": 2.961018208013367e-06, | |
| "loss": -0.0014, | |
| "step": 243 | |
| }, | |
| { | |
| "clip_ratio": 0.0015606118831783533, | |
| "completion_length": 740.5000305175781, | |
| "epoch": 0.004552026491301712, | |
| "grad_norm": 0.1023816242814064, | |
| "kl": 0.010467529296875, | |
| "learning_rate": 2.9465037218645694e-06, | |
| "loss": -0.0155, | |
| "num_tokens": 2764042.0, | |
| "reward": 0.062079211696982384, | |
| "reward_std": 0.04148745723068714, | |
| "rewards/code_reward": 0.000412541237892583, | |
| "rewards/format_reward": 0.6166666448116302, | |
| "step": 244 | |
| }, | |
| { | |
| "clip_ratio": 0.0012834252556785941, | |
| "epoch": 0.004570682337577538, | |
| "grad_norm": 0.09794943034648895, | |
| "kl": 0.01129150390625, | |
| "learning_rate": 2.9319809908131604e-06, | |
| "loss": -0.0154, | |
| "step": 245 | |
| }, | |
| { | |
| "clip_ratio": 0.0014966255985200405, | |
| "epoch": 0.004589338183853365, | |
| "grad_norm": 0.09920386970043182, | |
| "kl": 0.011505126953125, | |
| "learning_rate": 2.917450624203847e-06, | |
| "loss": -0.0159, | |
| "step": 246 | |
| }, | |
| { | |
| "clip_ratio": 0.0009015223768074065, | |
| "completion_length": 675.5416870117188, | |
| "epoch": 0.0046079940301291915, | |
| "grad_norm": 0.08201450854539871, | |
| "kl": 0.02886962890625, | |
| "learning_rate": 2.9029132317017118e-06, | |
| "loss": 0.0237, | |
| "num_tokens": 2792831.0, | |
| "reward": 0.4770751856267452, | |
| "reward_std": 0.21596147678792477, | |
| "rewards/code_reward": 0.41707515716552734, | |
| "rewards/format_reward": 0.5999999940395355, | |
| "step": 247 | |
| }, | |
| { | |
| "clip_ratio": 0.0009108328085858375, | |
| "epoch": 0.004626649876405018, | |
| "grad_norm": 0.0822167694568634, | |
| "kl": 0.0335693359375, | |
| "learning_rate": 2.888369423266629e-06, | |
| "loss": 0.0241, | |
| "step": 248 | |
| }, | |
| { | |
| "clip_ratio": 0.0011067874147556722, | |
| "epoch": 0.0046453057226808455, | |
| "grad_norm": 0.08312199264764786, | |
| "kl": 0.0335693359375, | |
| "learning_rate": 2.8738198091276712e-06, | |
| "loss": 0.0238, | |
| "step": 249 | |
| }, | |
| { | |
| "clip_ratio": 0.0007156841747928411, | |
| "completion_length": 705.1666870117188, | |
| "epoch": 0.004663961568956672, | |
| "grad_norm": 0.09860284626483917, | |
| "kl": 0.033416748046875, | |
| "learning_rate": 2.859264999757509e-06, | |
| "loss": 0.0451, | |
| "num_tokens": 2822631.0, | |
| "reward": 0.5245833247900009, | |
| "reward_std": 0.4078162908554077, | |
| "rewards/code_reward": 0.4583333358168602, | |
| "rewards/format_reward": 0.6624999940395355, | |
| "step": 250 | |
| }, | |
| { | |
| "clip_ratio": 0.0007305167673621327, | |
| "epoch": 0.004682617415232499, | |
| "grad_norm": 0.10016681998968124, | |
| "kl": 0.0286865234375, | |
| "learning_rate": 2.8447056058467928e-06, | |
| "loss": 0.0451, | |
| "step": 251 | |
| }, | |
| { | |
| "clip_ratio": 0.0007170947210397571, | |
| "epoch": 0.004701273261508325, | |
| "grad_norm": 0.09779990464448929, | |
| "kl": 0.029632568359375, | |
| "learning_rate": 2.830142238278531e-06, | |
| "loss": 0.045, | |
| "step": 252 | |
| }, | |
| { | |
| "clip_ratio": 0.000481658848002553, | |
| "completion_length": 590.2916870117188, | |
| "epoch": 0.004719929107784152, | |
| "grad_norm": 154.7192840576172, | |
| "kl": 92.0189208984375, | |
| "learning_rate": 2.81557550810246e-06, | |
| "loss": 0.9582, | |
| "num_tokens": 2852470.0, | |
| "reward": 0.5958333238959312, | |
| "reward_std": 0.3366524577140808, | |
| "rewards/code_reward": 0.541666679084301, | |
| "rewards/format_reward": 0.5416666716337204, | |
| "step": 253 | |
| }, | |
| { | |
| "clip_ratio": 0.0004959633661201224, | |
| "epoch": 0.004738584954059978, | |
| "grad_norm": 25.69236946105957, | |
| "kl": 15.9564208984375, | |
| "learning_rate": 2.8010060265094026e-06, | |
| "loss": 0.1926, | |
| "step": 254 | |
| }, | |
| { | |
| "clip_ratio": 0.0010467918764334172, | |
| "epoch": 0.004757240800335805, | |
| "grad_norm": 2.245051383972168, | |
| "kl": 1.70458984375, | |
| "learning_rate": 2.786434404805629e-06, | |
| "loss": 0.0514, | |
| "step": 255 | |
| }, | |
| { | |
| "clip_ratio": 0.0009520899475319311, | |
| "completion_length": 900.1250305175781, | |
| "epoch": 0.004775896646611632, | |
| "grad_norm": 0.0822591632604599, | |
| "kl": 0.01397705078125, | |
| "learning_rate": 2.771861254387199e-06, | |
| "loss": -0.0279, | |
| "num_tokens": 2891485.0, | |
| "reward": 0.3816172480583191, | |
| "reward_std": 0.25385507196187973, | |
| "rewards/code_reward": 0.3220338970422745, | |
| "rewards/format_reward": 0.5958333313465118, | |
| "step": 256 | |
| }, | |
| { | |
| "clip_ratio": 0.001093231316190213, | |
| "epoch": 0.004794552492887459, | |
| "grad_norm": 0.08108334243297577, | |
| "kl": 0.01324462890625, | |
| "learning_rate": 2.7572871867143204e-06, | |
| "loss": -0.0278, | |
| "step": 257 | |
| }, | |
| { | |
| "clip_ratio": 0.0013529035350074992, | |
| "epoch": 0.004813208339163285, | |
| "grad_norm": 0.07985206693410873, | |
| "kl": 0.01263427734375, | |
| "learning_rate": 2.742712813285681e-06, | |
| "loss": -0.0277, | |
| "step": 258 | |
| }, | |
| { | |
| "clip_ratio": 0.001162135595222935, | |
| "completion_length": 1206.5833740234375, | |
| "epoch": 0.004831864185439112, | |
| "grad_norm": 0.06389827281236649, | |
| "kl": 0.0082855224609375, | |
| "learning_rate": 2.7281387456128017e-06, | |
| "loss": -0.0507, | |
| "num_tokens": 2938767.0, | |
| "reward": 0.10169872269034386, | |
| "reward_std": 0.16118879988789558, | |
| "rewards/code_reward": 0.051282052882015705, | |
| "rewards/format_reward": 0.5041666701436043, | |
| "step": 259 | |
| }, | |
| { | |
| "clip_ratio": 0.0009780275577213615, | |
| "epoch": 0.0048505200317149385, | |
| "grad_norm": 0.06325838714838028, | |
| "kl": 0.008026123046875, | |
| "learning_rate": 2.7135655951943716e-06, | |
| "loss": -0.0507, | |
| "step": 260 | |
| }, | |
| { | |
| "clip_ratio": 0.0008303393551614136, | |
| "epoch": 0.004869175877990765, | |
| "grad_norm": 0.06411907821893692, | |
| "kl": 0.00754547119140625, | |
| "learning_rate": 2.698993973490598e-06, | |
| "loss": -0.0507, | |
| "step": 261 | |
| }, | |
| { | |
| "clip_ratio": 0.0009404766315128654, | |
| "completion_length": 799.3333435058594, | |
| "epoch": 0.004887831724266592, | |
| "grad_norm": 0.07781538367271423, | |
| "kl": 0.00923919677734375, | |
| "learning_rate": 2.6844244918975416e-06, | |
| "loss": 0.0603, | |
| "num_tokens": 2969987.0, | |
| "reward": 0.532083310186863, | |
| "reward_std": 0.1838482804596424, | |
| "rewards/code_reward": 0.4583333432674408, | |
| "rewards/format_reward": 0.737500011920929, | |
| "step": 262 | |
| }, | |
| { | |
| "clip_ratio": 0.0007310473010875285, | |
| "epoch": 0.004906487570542419, | |
| "grad_norm": 0.08621031045913696, | |
| "kl": 0.0091094970703125, | |
| "learning_rate": 2.66985776172147e-06, | |
| "loss": 0.0604, | |
| "step": 263 | |
| }, | |
| { | |
| "clip_ratio": 0.0009290947928093374, | |
| "epoch": 0.004925143416818246, | |
| "grad_norm": 0.07831931114196777, | |
| "kl": 0.00958251953125, | |
| "learning_rate": 2.6552943941532088e-06, | |
| "loss": 0.0602, | |
| "step": 264 | |
| }, | |
| { | |
| "clip_ratio": 0.0010500443167984486, | |
| "completion_length": 902.4167175292969, | |
| "epoch": 0.004943799263094072, | |
| "grad_norm": 0.09909704327583313, | |
| "kl": 0.013946533203125, | |
| "learning_rate": 2.6407350002424927e-06, | |
| "loss": -0.0336, | |
| "num_tokens": 3008697.0, | |
| "reward": 0.30541668087244034, | |
| "reward_std": 0.2829319443553686, | |
| "rewards/code_reward": 0.25, | |
| "rewards/format_reward": 0.5541666597127914, | |
| "step": 265 | |
| }, | |
| { | |
| "clip_ratio": 0.0010122012172359973, | |
| "epoch": 0.004962455109369899, | |
| "grad_norm": 0.07800783216953278, | |
| "kl": 0.014617919921875, | |
| "learning_rate": 2.626180190872329e-06, | |
| "loss": -0.0339, | |
| "step": 266 | |
| }, | |
| { | |
| "clip_ratio": 0.0009586416417732835, | |
| "epoch": 0.004981110955645725, | |
| "grad_norm": 0.08270706981420517, | |
| "kl": 0.013427734375, | |
| "learning_rate": 2.611630576733372e-06, | |
| "loss": -0.034, | |
| "step": 267 | |
| }, | |
| { | |
| "clip_ratio": 0.0006794014479964972, | |
| "completion_length": 852.8750610351562, | |
| "epoch": 0.004999766801921552, | |
| "grad_norm": 0.0892496407032013, | |
| "kl": 0.01239013671875, | |
| "learning_rate": 2.5970867682982885e-06, | |
| "loss": -0.0121, | |
| "num_tokens": 3046734.0, | |
| "reward": 0.6499094069004059, | |
| "reward_std": 0.21050915122032166, | |
| "rewards/code_reward": 0.5765760540962219, | |
| "rewards/format_reward": 0.7333333194255829, | |
| "step": 268 | |
| }, | |
| { | |
| "clip_ratio": 0.0004855144943576306, | |
| "epoch": 0.005018422648197379, | |
| "grad_norm": 0.08327967673540115, | |
| "kl": 0.011993408203125, | |
| "learning_rate": 2.582549375796154e-06, | |
| "loss": -0.0124, | |
| "step": 269 | |
| }, | |
| { | |
| "clip_ratio": 0.000626907596597448, | |
| "epoch": 0.005037078494473206, | |
| "grad_norm": 0.08495430648326874, | |
| "kl": 0.0126953125, | |
| "learning_rate": 2.568019009186841e-06, | |
| "loss": -0.0121, | |
| "step": 270 | |
| }, | |
| { | |
| "clip_ratio": 0.0007202730048447847, | |
| "completion_length": 977.8333435058594, | |
| "epoch": 0.005055734340749032, | |
| "grad_norm": 0.09023862332105637, | |
| "kl": 0.024383544921875, | |
| "learning_rate": 2.5534962781354317e-06, | |
| "loss": 0.0087, | |
| "num_tokens": 3088262.0, | |
| "reward": 0.26893792301416397, | |
| "reward_std": 0.38880206644535065, | |
| "rewards/code_reward": 0.19893791526556015, | |
| "rewards/format_reward": 0.7000000178813934, | |
| "step": 271 | |
| }, | |
| { | |
| "clip_ratio": 0.0009908578940667212, | |
| "epoch": 0.005074390187024859, | |
| "grad_norm": 0.09044025838375092, | |
| "kl": 0.0234375, | |
| "learning_rate": 2.538981791986634e-06, | |
| "loss": 0.0088, | |
| "step": 272 | |
| }, | |
| { | |
| "clip_ratio": 0.0006488285725936294, | |
| "epoch": 0.0050930460333006855, | |
| "grad_norm": 0.0915515273809433, | |
| "kl": 0.022735595703125, | |
| "learning_rate": 2.524476159739218e-06, | |
| "loss": 0.0086, | |
| "step": 273 | |
| }, | |
| { | |
| "clip_ratio": 0.00046395487152040005, | |
| "completion_length": 707.9583740234375, | |
| "epoch": 0.005111701879576512, | |
| "grad_norm": 0.08589129149913788, | |
| "kl": 0.02691650390625, | |
| "learning_rate": 2.5099799900204607e-06, | |
| "loss": -0.013, | |
| "num_tokens": 3119713.0, | |
| "reward": 0.7045832872390747, | |
| "reward_std": 0.39258189499378204, | |
| "rewards/code_reward": 0.6250000149011612, | |
| "rewards/format_reward": 0.7958333194255829, | |
| "step": 274 | |
| }, | |
| { | |
| "clip_ratio": 0.0006349159230012447, | |
| "epoch": 0.005130357725852339, | |
| "grad_norm": 0.08367536962032318, | |
| "kl": 0.02716064453125, | |
| "learning_rate": 2.4954938910606108e-06, | |
| "loss": -0.013, | |
| "step": 275 | |
| }, | |
| { | |
| "clip_ratio": 0.0007932094158604741, | |
| "epoch": 0.005149013572128166, | |
| "grad_norm": 0.08294366300106049, | |
| "kl": 0.024322509765625, | |
| "learning_rate": 2.481018470667368e-06, | |
| "loss": -0.0131, | |
| "step": 276 | |
| }, | |
| { | |
| "clip_ratio": 0.0012548710801638663, | |
| "completion_length": 581.9166870117188, | |
| "epoch": 0.005167669418403993, | |
| "grad_norm": 0.09755374491214752, | |
| "kl": 0.02392578125, | |
| "learning_rate": 2.4665543362003802e-06, | |
| "loss": 0.0183, | |
| "num_tokens": 3144827.0, | |
| "reward": 0.7179166376590729, | |
| "reward_std": 0.27089906856417656, | |
| "rewards/code_reward": 0.6666666716337204, | |
| "rewards/format_reward": 0.5124999731779099, | |
| "step": 277 | |
| }, | |
| { | |
| "clip_ratio": 0.001595219480805099, | |
| "epoch": 0.005186325264679819, | |
| "grad_norm": 0.09835170209407806, | |
| "kl": 0.0220947265625, | |
| "learning_rate": 2.4521020945457615e-06, | |
| "loss": 0.0181, | |
| "step": 278 | |
| }, | |
| { | |
| "clip_ratio": 0.0019812395912595093, | |
| "epoch": 0.005204981110955646, | |
| "grad_norm": 0.0999624952673912, | |
| "kl": 0.0208740234375, | |
| "learning_rate": 2.4376623520906255e-06, | |
| "loss": 0.0182, | |
| "step": 279 | |
| }, | |
| { | |
| "clip_ratio": 0.0011003862309735268, | |
| "completion_length": 1171.5833435058594, | |
| "epoch": 0.005223636957231472, | |
| "grad_norm": 0.07472976297140121, | |
| "kl": 0.009613037109375, | |
| "learning_rate": 2.4232357146976478e-06, | |
| "loss": 0.0011, | |
| "num_tokens": 3189013.0, | |
| "reward": 0.5404510945081711, | |
| "reward_std": 0.15828495100140572, | |
| "rewards/code_reward": 0.4721178039908409, | |
| "rewards/format_reward": 0.6833333224058151, | |
| "step": 280 | |
| }, | |
| { | |
| "clip_ratio": 0.0007794810517225415, | |
| "epoch": 0.005242292803507299, | |
| "grad_norm": 0.07738126814365387, | |
| "kl": 0.009857177734375, | |
| "learning_rate": 2.408822787679637e-06, | |
| "loss": 0.0013, | |
| "step": 281 | |
| }, | |
| { | |
| "clip_ratio": 0.0008492782071698457, | |
| "epoch": 0.005260948649783125, | |
| "grad_norm": 0.07533033192157745, | |
| "kl": 0.009613037109375, | |
| "learning_rate": 2.3944241757741475e-06, | |
| "loss": 0.001, | |
| "step": 282 | |
| }, | |
| { | |
| "clip_ratio": 0.0003291434477432631, | |
| "completion_length": 672.625, | |
| "epoch": 0.005279604496058953, | |
| "grad_norm": 0.07457376271486282, | |
| "kl": 0.012908935546875, | |
| "learning_rate": 2.380040483118097e-06, | |
| "loss": -0.0139, | |
| "num_tokens": 3212848.0, | |
| "reward": 1.0279166400432587, | |
| "reward_std": 0.17488877102732658, | |
| "rewards/code_reward": 0.9583333432674408, | |
| "rewards/format_reward": 0.6958333253860474, | |
| "step": 283 | |
| }, | |
| { | |
| "clip_ratio": 5.457324004964903e-05, | |
| "epoch": 0.005298260342334779, | |
| "grad_norm": 0.07336500287055969, | |
| "kl": 0.0133056640625, | |
| "learning_rate": 2.365672313222419e-06, | |
| "loss": -0.0141, | |
| "step": 284 | |
| }, | |
| { | |
| "clip_ratio": 0.0002531533937144559, | |
| "epoch": 0.005316916188610606, | |
| "grad_norm": 0.07265036553144455, | |
| "kl": 0.0135498046875, | |
| "learning_rate": 2.351320268946749e-06, | |
| "loss": -0.014, | |
| "step": 285 | |
| }, | |
| { | |
| "clip_ratio": 0.0013167554279789329, | |
| "completion_length": 909.7083740234375, | |
| "epoch": 0.0053355720348864325, | |
| "grad_norm": 0.11532127857208252, | |
| "kl": 0.01422119140625, | |
| "learning_rate": 2.336984952474119e-06, | |
| "loss": -0.0194, | |
| "num_tokens": 3247833.0, | |
| "reward": 0.09151961281895638, | |
| "reward_std": 0.09498994797468185, | |
| "rewards/code_reward": 0.04901960864663124, | |
| "rewards/format_reward": 0.42499999701976776, | |
| "step": 286 | |
| }, | |
| { | |
| "clip_ratio": 0.0011718517343979329, | |
| "epoch": 0.005354227881162259, | |
| "grad_norm": 0.10550913959741592, | |
| "kl": 0.013946533203125, | |
| "learning_rate": 2.322666965285697e-06, | |
| "loss": -0.0197, | |
| "step": 287 | |
| }, | |
| { | |
| "clip_ratio": 0.0012868313351646066, | |
| "epoch": 0.005372883727438086, | |
| "grad_norm": 0.10481560975313187, | |
| "kl": 0.015716552734375, | |
| "learning_rate": 2.3083669081355507e-06, | |
| "loss": -0.0198, | |
| "step": 288 | |
| }, | |
| { | |
| "clip_ratio": 0.00034923299972433597, | |
| "completion_length": 1332.0833740234375, | |
| "epoch": 0.005391539573713912, | |
| "grad_norm": 0.06407570093870163, | |
| "kl": 0.0085906982421875, | |
| "learning_rate": 2.2940853810254377e-06, | |
| "loss": 0.0465, | |
| "num_tokens": 3299579.0, | |
| "reward": 0.05910714715719223, | |
| "reward_std": 0.04962867684662342, | |
| "rewards/code_reward": 0.0053571430034935474, | |
| "rewards/format_reward": 0.5374999940395355, | |
| "step": 289 | |
| }, | |
| { | |
| "clip_ratio": 0.0005735456070397049, | |
| "epoch": 0.00541019541998974, | |
| "grad_norm": 0.08031246811151505, | |
| "kl": 0.00830078125, | |
| "learning_rate": 2.2798229831796313e-06, | |
| "loss": 0.0464, | |
| "step": 290 | |
| }, | |
| { | |
| "clip_ratio": 0.0007879029144532979, | |
| "epoch": 0.005428851266265566, | |
| "grad_norm": 0.0649939477443695, | |
| "kl": 0.0087432861328125, | |
| "learning_rate": 2.2655803130197816e-06, | |
| "loss": 0.0464, | |
| "step": 291 | |
| }, | |
| { | |
| "clip_ratio": 0.00107748550362885, | |
| "completion_length": 1028.0833740234375, | |
| "epoch": 0.005447507112541393, | |
| "grad_norm": 0.10151123255491257, | |
| "kl": 0.0071563720703125, | |
| "learning_rate": 2.2513579681398034e-06, | |
| "loss": 0.0503, | |
| "num_tokens": 3348829.0, | |
| "reward": 0.1927850916981697, | |
| "reward_std": 0.3257390707731247, | |
| "rewards/code_reward": 0.13903508707880974, | |
| "rewards/format_reward": 0.5375000089406967, | |
| "step": 292 | |
| }, | |
| { | |
| "clip_ratio": 0.0008358334598597139, | |
| "epoch": 0.005466162958817219, | |
| "grad_norm": 0.09055967628955841, | |
| "kl": 0.0071868896484375, | |
| "learning_rate": 2.237156545280803e-06, | |
| "loss": 0.0504, | |
| "step": 293 | |
| }, | |
| { | |
| "clip_ratio": 0.0009988098172470927, | |
| "epoch": 0.005484818805093046, | |
| "grad_norm": 0.08410565555095673, | |
| "kl": 0.0072174072265625, | |
| "learning_rate": 2.2229766403060403e-06, | |
| "loss": 0.0501, | |
| "step": 294 | |
| }, | |
| { | |
| "clip_ratio": 0.001458263781387359, | |
| "completion_length": 681.2083740234375, | |
| "epoch": 0.005503474651368872, | |
| "grad_norm": 0.09818808734416962, | |
| "kl": 0.01514434814453125, | |
| "learning_rate": 2.2088188481759305e-06, | |
| "loss": 0.0187, | |
| "num_tokens": 3378258.0, | |
| "reward": 0.5579166747629642, | |
| "reward_std": 0.045613983646035194, | |
| "rewards/code_reward": 0.5, | |
| "rewards/format_reward": 0.5791666805744171, | |
| "step": 295 | |
| }, | |
| { | |
| "clip_ratio": 0.0013955137401353568, | |
| "epoch": 0.005522130497644699, | |
| "grad_norm": 0.09592823684215546, | |
| "kl": 0.017864227294921875, | |
| "learning_rate": 2.194683762923073e-06, | |
| "loss": 0.0188, | |
| "step": 296 | |
| }, | |
| { | |
| "clip_ratio": 0.0017046461580321193, | |
| "epoch": 0.005540786343920526, | |
| "grad_norm": 0.09802346676588058, | |
| "kl": 0.01699066162109375, | |
| "learning_rate": 2.1805719776273387e-06, | |
| "loss": 0.0187, | |
| "step": 297 | |
| }, | |
| { | |
| "clip_ratio": 0.0009031847293954343, | |
| "completion_length": 919.2917175292969, | |
| "epoch": 0.005559442190196353, | |
| "grad_norm": 0.08558425307273865, | |
| "kl": 0.012786865234375, | |
| "learning_rate": 2.166484084390974e-06, | |
| "loss": -0.0303, | |
| "num_tokens": 3419881.0, | |
| "reward": 0.5293938517570496, | |
| "reward_std": 0.3867318332195282, | |
| "rewards/code_reward": 0.4823105186223984, | |
| "rewards/format_reward": 0.47083334624767303, | |
| "step": 298 | |
| }, | |
| { | |
| "clip_ratio": 0.0009096206922549754, | |
| "epoch": 0.0055780980364721795, | |
| "grad_norm": 0.0866839662194252, | |
| "kl": 0.0133056640625, | |
| "learning_rate": 2.1524206743137636e-06, | |
| "loss": -0.0303, | |
| "step": 299 | |
| }, | |
| { | |
| "clip_ratio": 0.00072934762283694, | |
| "epoch": 0.005596753882748006, | |
| "grad_norm": 0.08566750586032867, | |
| "kl": 0.0135498046875, | |
| "learning_rate": 2.1383823374682287e-06, | |
| "loss": -0.0304, | |
| "step": 300 | |
| }, | |
| { | |
| "clip_ratio": 0.0010719166893977672, | |
| "completion_length": 849.1666870117188, | |
| "epoch": 0.005615409729023833, | |
| "grad_norm": 0.09118139743804932, | |
| "kl": 0.0116424560546875, | |
| "learning_rate": 2.124369662874868e-06, | |
| "loss": 0.0616, | |
| "num_tokens": 3455405.0, | |
| "reward": 0.5117070078849792, | |
| "reward_std": 0.4537804424762726, | |
| "rewards/code_reward": 0.43629033863544464, | |
| "rewards/format_reward": 0.7541666626930237, | |
| "step": 301 | |
| }, | |
| { | |
| "clip_ratio": 0.0009527777438051999, | |
| "epoch": 0.005634065575299659, | |
| "grad_norm": 0.09360919892787933, | |
| "kl": 0.0117034912109375, | |
| "learning_rate": 2.110383238477441e-06, | |
| "loss": 0.0618, | |
| "step": 302 | |
| }, | |
| { | |
| "clip_ratio": 0.0010622217669151723, | |
| "epoch": 0.005652721421575487, | |
| "grad_norm": 0.09148411452770233, | |
| "kl": 0.0118560791015625, | |
| "learning_rate": 2.096423651118305e-06, | |
| "loss": 0.0616, | |
| "step": 303 | |
| }, | |
| { | |
| "clip_ratio": 0.0005411781894508749, | |
| "completion_length": 1098.7083740234375, | |
| "epoch": 0.005671377267851313, | |
| "grad_norm": 0.08525364845991135, | |
| "kl": 0.014556884765625, | |
| "learning_rate": 2.082491486513788e-06, | |
| "loss": 0.0016, | |
| "num_tokens": 3502546.0, | |
| "reward": 0.3728175610303879, | |
| "reward_std": 0.3381016403436661, | |
| "rewards/code_reward": 0.321984238922596, | |
| "rewards/format_reward": 0.5083333253860474, | |
| "step": 304 | |
| }, | |
| { | |
| "clip_ratio": 0.0007919937779661268, | |
| "epoch": 0.00569003311412714, | |
| "grad_norm": 0.40698713064193726, | |
| "kl": 0.01470947265625, | |
| "learning_rate": 2.0685873292296116e-06, | |
| "loss": 0.0018, | |
| "step": 305 | |
| }, | |
| { | |
| "clip_ratio": 0.0005383995739975944, | |
| "epoch": 0.005708688960402966, | |
| "grad_norm": 0.49188467860221863, | |
| "kl": 0.014678955078125, | |
| "learning_rate": 2.054711762656369e-06, | |
| "loss": 0.0018, | |
| "step": 306 | |
| }, | |
| { | |
| "clip_ratio": 0.0010625849245116115, | |
| "completion_length": 683.9583435058594, | |
| "epoch": 0.005727344806678793, | |
| "grad_norm": 0.10668221861124039, | |
| "kl": 0.014434814453125, | |
| "learning_rate": 2.040865368985044e-06, | |
| "loss": -0.0085, | |
| "num_tokens": 3530901.0, | |
| "reward": 0.776462584733963, | |
| "reward_std": 0.17944780550897121, | |
| "rewards/code_reward": 0.7243793159723282, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 307 | |
| }, | |
| { | |
| "clip_ratio": 0.0016164338449016213, | |
| "epoch": 0.005746000652954619, | |
| "grad_norm": 0.10843304544687271, | |
| "kl": 0.0142822265625, | |
| "learning_rate": 2.027048729182583e-06, | |
| "loss": -0.0086, | |
| "step": 308 | |
| }, | |
| { | |
| "clip_ratio": 0.001167797192465514, | |
| "epoch": 0.005764656499230446, | |
| "grad_norm": 0.10622025281190872, | |
| "kl": 0.014434814453125, | |
| "learning_rate": 2.0132624229675205e-06, | |
| "loss": -0.0089, | |
| "step": 309 | |
| }, | |
| { | |
| "clip_ratio": 0.0011982419237028807, | |
| "completion_length": 876.9166870117188, | |
| "epoch": 0.005783312345506273, | |
| "grad_norm": 0.09164712578058243, | |
| "kl": 0.021636962890625, | |
| "learning_rate": 1.9995070287856546e-06, | |
| "loss": -0.0053, | |
| "num_tokens": 3564103.0, | |
| "reward": 0.5786574631929398, | |
| "reward_std": 0.41006001830101013, | |
| "rewards/code_reward": 0.5207407623529434, | |
| "rewards/format_reward": 0.5791666507720947, | |
| "step": 310 | |
| }, | |
| { | |
| "clip_ratio": 0.001371737860608846, | |
| "epoch": 0.0058019681917821, | |
| "grad_norm": 0.09242158383131027, | |
| "kl": 0.022064208984375, | |
| "learning_rate": 1.985783123785774e-06, | |
| "loss": -0.0055, | |
| "step": 311 | |
| }, | |
| { | |
| "clip_ratio": 0.001422962493961677, | |
| "epoch": 0.0058206240380579265, | |
| "grad_norm": 0.09199241548776627, | |
| "kl": 0.021728515625, | |
| "learning_rate": 1.9720912837954486e-06, | |
| "loss": -0.0054, | |
| "step": 312 | |
| }, | |
| { | |
| "clip_ratio": 0.0006902147724758834, | |
| "completion_length": 927.7500305175781, | |
| "epoch": 0.005839279884333753, | |
| "grad_norm": 0.09657740592956543, | |
| "kl": 0.0142822265625, | |
| "learning_rate": 1.958432083296862e-06, | |
| "loss": -0.0334, | |
| "num_tokens": 3595285.0, | |
| "reward": 0.4076350927352905, | |
| "reward_std": 0.3373851850628853, | |
| "rewards/code_reward": 0.3297183997929096, | |
| "rewards/format_reward": 0.7791666686534882, | |
| "step": 313 | |
| }, | |
| { | |
| "clip_ratio": 0.0006798989197704941, | |
| "epoch": 0.00585793573060958, | |
| "grad_norm": 0.09554401785135269, | |
| "kl": 0.013671875, | |
| "learning_rate": 1.9448060954027093e-06, | |
| "loss": -0.0335, | |
| "step": 314 | |
| }, | |
| { | |
| "clip_ratio": 0.0008473133784718812, | |
| "epoch": 0.005876591576885406, | |
| "grad_norm": 0.09273131936788559, | |
| "kl": 0.014678955078125, | |
| "learning_rate": 1.931213891832153e-06, | |
| "loss": -0.0337, | |
| "step": 315 | |
| }, | |
| { | |
| "clip_ratio": 0.0005323913064785302, | |
| "completion_length": 834.8333740234375, | |
| "epoch": 0.005895247423161233, | |
| "grad_norm": 0.10354404151439667, | |
| "kl": 0.025360107421875, | |
| "learning_rate": 1.9176560428868336e-06, | |
| "loss": -0.0085, | |
| "num_tokens": 3639501.0, | |
| "reward": 0.64698725938797, | |
| "reward_std": 0.4041699767112732, | |
| "rewards/code_reward": 0.5749039202928543, | |
| "rewards/format_reward": 0.7208333015441895, | |
| "step": 316 | |
| }, | |
| { | |
| "clip_ratio": 0.0008116684039123356, | |
| "epoch": 0.00591390326943706, | |
| "grad_norm": 0.10160063952207565, | |
| "kl": 0.025299072265625, | |
| "learning_rate": 1.9041331174269373e-06, | |
| "loss": -0.0088, | |
| "step": 317 | |
| }, | |
| { | |
| "clip_ratio": 0.0005085803277324885, | |
| "epoch": 0.005932559115712887, | |
| "grad_norm": 0.09548759460449219, | |
| "kl": 0.0244140625, | |
| "learning_rate": 1.8906456828473341e-06, | |
| "loss": -0.009, | |
| "step": 318 | |
| }, | |
| { | |
| "clip_ratio": 0.0008387799316551536, | |
| "completion_length": 1620.8750915527344, | |
| "epoch": 0.005951214961988713, | |
| "grad_norm": 0.07884962111711502, | |
| "kl": 0.0074615478515625, | |
| "learning_rate": 1.8771943050537656e-06, | |
| "loss": 0.0221, | |
| "num_tokens": 3698802.0, | |
| "reward": 0.38234737887978554, | |
| "reward_std": 0.20284880511462688, | |
| "rewards/code_reward": 0.30693069100379944, | |
| "rewards/format_reward": 0.7541666626930237, | |
| "step": 319 | |
| }, | |
| { | |
| "clip_ratio": 0.0007986939453985542, | |
| "epoch": 0.00596987080826454, | |
| "grad_norm": 0.07618702203035355, | |
| "kl": 0.007415771484375, | |
| "learning_rate": 1.8637795484391046e-06, | |
| "loss": 0.0221, | |
| "step": 320 | |
| }, | |
| { | |
| "clip_ratio": 0.0007583920087199658, | |
| "epoch": 0.0059885266545403664, | |
| "grad_norm": 0.0829911157488823, | |
| "kl": 0.0075225830078125, | |
| "learning_rate": 1.8504019758596698e-06, | |
| "loss": 0.0221, | |
| "step": 321 | |
| }, | |
| { | |
| "clip_ratio": 0.001169055758509785, | |
| "completion_length": 795.7916717529297, | |
| "epoch": 0.006007182500816193, | |
| "grad_norm": 0.14611609280109406, | |
| "kl": 0.019012451171875, | |
| "learning_rate": 1.8370621486116163e-06, | |
| "loss": 0.023, | |
| "num_tokens": 3734797.0, | |
| "reward": 0.645012378692627, | |
| "reward_std": 0.2191050536930561, | |
| "rewards/code_reward": 0.5820957124233246, | |
| "rewards/format_reward": 0.6291666924953461, | |
| "step": 322 | |
| }, | |
| { | |
| "clip_ratio": 0.0009497803403064609, | |
| "epoch": 0.0060258383470920196, | |
| "grad_norm": 0.10970307886600494, | |
| "kl": 0.02032470703125, | |
| "learning_rate": 1.823760626407377e-06, | |
| "loss": 0.0227, | |
| "step": 323 | |
| }, | |
| { | |
| "clip_ratio": 0.0009026691550388932, | |
| "epoch": 0.006044494193367847, | |
| "grad_norm": 0.11829708516597748, | |
| "kl": 0.0189208984375, | |
| "learning_rate": 1.8104979673521838e-06, | |
| "loss": 0.023, | |
| "step": 324 | |
| }, | |
| { | |
| "clip_ratio": 0.0008327370742335916, | |
| "completion_length": 874.0416870117188, | |
| "epoch": 0.0060631500396436736, | |
| "grad_norm": 0.09914888441562653, | |
| "kl": 0.02264404296875, | |
| "learning_rate": 1.7972747279206482e-06, | |
| "loss": -0.0067, | |
| "num_tokens": 3769226.0, | |
| "reward": 0.4188089519739151, | |
| "reward_std": 0.19916179403662682, | |
| "rewards/code_reward": 0.36964227352291346, | |
| "rewards/format_reward": 0.49166665971279144, | |
| "step": 325 | |
| }, | |
| { | |
| "clip_ratio": 0.0008456225623376667, | |
| "epoch": 0.0060818058859195, | |
| "grad_norm": 0.09671460092067719, | |
| "kl": 0.02288818359375, | |
| "learning_rate": 1.7840914629334122e-06, | |
| "loss": -0.0064, | |
| "step": 326 | |
| }, | |
| { | |
| "clip_ratio": 0.0010657160892151296, | |
| "epoch": 0.006100461732195327, | |
| "grad_norm": 0.09105102717876434, | |
| "kl": 0.02294921875, | |
| "learning_rate": 1.7709487255338731e-06, | |
| "loss": -0.0066, | |
| "step": 327 | |
| }, | |
| { | |
| "clip_ratio": 0.0010840718168765306, | |
| "completion_length": 422.5833435058594, | |
| "epoch": 0.006119117578471153, | |
| "grad_norm": 0.13927879929542542, | |
| "kl": 0.019073486328125, | |
| "learning_rate": 1.7578470671649684e-06, | |
| "loss": 0.0094, | |
| "num_tokens": 3789448.0, | |
| "reward": 0.5294444337487221, | |
| "reward_std": 0.17376149259507656, | |
| "rewards/code_reward": 0.4444444179534912, | |
| "rewards/format_reward": 0.8499999940395355, | |
| "step": 328 | |
| }, | |
| { | |
| "clip_ratio": 0.000780029222369194, | |
| "epoch": 0.00613777342474698, | |
| "grad_norm": 0.13586357235908508, | |
| "kl": 0.021209716796875, | |
| "learning_rate": 1.744787037546045e-06, | |
| "loss": 0.0092, | |
| "step": 329 | |
| }, | |
| { | |
| "clip_ratio": 0.001488116628024727, | |
| "epoch": 0.006156429271022806, | |
| "grad_norm": 0.16369539499282837, | |
| "kl": 0.02215576171875, | |
| "learning_rate": 1.731769184649788e-06, | |
| "loss": 0.0091, | |
| "step": 330 | |
| }, | |
| { | |
| "clip_ratio": 0.0005380379006965086, | |
| "completion_length": 776.1250305175781, | |
| "epoch": 0.006175085117298634, | |
| "grad_norm": 0.0828891322016716, | |
| "kl": 0.02215576171875, | |
| "learning_rate": 1.7187940546792325e-06, | |
| "loss": -0.0154, | |
| "num_tokens": 3825619.0, | |
| "reward": 0.6713247746229172, | |
| "reward_std": 0.09931758604943752, | |
| "rewards/code_reward": 0.5779914557933807, | |
| "rewards/format_reward": 0.9333333075046539, | |
| "step": 331 | |
| }, | |
| { | |
| "clip_ratio": 0.0008244940836448222, | |
| "epoch": 0.00619374096357446, | |
| "grad_norm": 0.08281542360782623, | |
| "kl": 0.02386474609375, | |
| "learning_rate": 1.7058621920448465e-06, | |
| "loss": -0.0155, | |
| "step": 332 | |
| }, | |
| { | |
| "clip_ratio": 0.0006863593007437885, | |
| "epoch": 0.006212396809850287, | |
| "grad_norm": 0.08620591461658478, | |
| "kl": 0.027679443359375, | |
| "learning_rate": 1.6929741393416855e-06, | |
| "loss": -0.0155, | |
| "step": 333 | |
| }, | |
| { | |
| "clip_ratio": 0.0011899169767275453, | |
| "completion_length": 866.6250305175781, | |
| "epoch": 0.0062310526561261135, | |
| "grad_norm": 0.09219539165496826, | |
| "kl": 0.034088134765625, | |
| "learning_rate": 1.6801304373266286e-06, | |
| "loss": 0.0009, | |
| "num_tokens": 3860470.0, | |
| "reward": 0.526757076382637, | |
| "reward_std": 0.33820731937885284, | |
| "rewards/code_reward": 0.45467372983694077, | |
| "rewards/format_reward": 0.7208333313465118, | |
| "step": 334 | |
| }, | |
| { | |
| "clip_ratio": 0.0009668752900324762, | |
| "epoch": 0.00624970850240194, | |
| "grad_norm": 0.09143849462270737, | |
| "kl": 0.030029296875, | |
| "learning_rate": 1.667331624895689e-06, | |
| "loss": 0.0004, | |
| "step": 335 | |
| }, | |
| { | |
| "clip_ratio": 0.0009758240194059908, | |
| "epoch": 0.006268364348677767, | |
| "grad_norm": 0.09234081953763962, | |
| "kl": 0.03564453125, | |
| "learning_rate": 1.6545782390614037e-06, | |
| "loss": 0.0005, | |
| "step": 336 | |
| }, | |
| { | |
| "clip_ratio": 0.0009786576265469193, | |
| "completion_length": 767.2083740234375, | |
| "epoch": 0.006287020194953594, | |
| "grad_norm": 0.11128581315279007, | |
| "kl": 0.0589599609375, | |
| "learning_rate": 1.6418708149302992e-06, | |
| "loss": 0.0293, | |
| "num_tokens": 3900099.0, | |
| "reward": 0.5688367038965225, | |
| "reward_std": 0.1789155676960945, | |
| "rewards/code_reward": 0.49841997027397156, | |
| "rewards/format_reward": 0.7041666805744171, | |
| "step": 337 | |
| }, | |
| { | |
| "clip_ratio": 0.0011234881239943206, | |
| "epoch": 0.0063056760412294206, | |
| "grad_norm": 0.11240074783563614, | |
| "kl": 0.0528564453125, | |
| "learning_rate": 1.6292098856804423e-06, | |
| "loss": 0.029, | |
| "step": 338 | |
| }, | |
| { | |
| "clip_ratio": 0.0006711997266393155, | |
| "epoch": 0.006324331887505247, | |
| "grad_norm": 0.10572442412376404, | |
| "kl": 0.05133056640625, | |
| "learning_rate": 1.6165959825390661e-06, | |
| "loss": 0.0289, | |
| "step": 339 | |
| }, | |
| { | |
| "clip_ratio": 0.0007651688065379858, | |
| "completion_length": 662.3333587646484, | |
| "epoch": 0.006342987733781074, | |
| "grad_norm": 0.09474088996648788, | |
| "kl": 0.0198974609375, | |
| "learning_rate": 1.604029634760284e-06, | |
| "loss": 0.0135, | |
| "num_tokens": 3926987.0, | |
| "reward": 0.8191105425357819, | |
| "reward_std": 0.3644227534532547, | |
| "rewards/code_reward": 0.7049438655376434, | |
| "rewards/format_reward": 1.1416666507720947, | |
| "step": 340 | |
| }, | |
| { | |
| "clip_ratio": 0.0006743039994034916, | |
| "epoch": 0.0063616435800569, | |
| "grad_norm": 0.09659333527088165, | |
| "kl": 0.020263671875, | |
| "learning_rate": 1.59151136960288e-06, | |
| "loss": 0.0139, | |
| "step": 341 | |
| }, | |
| { | |
| "clip_ratio": 0.0006922298634890467, | |
| "epoch": 0.006380299426332727, | |
| "grad_norm": 0.09575100243091583, | |
| "kl": 0.01959228515625, | |
| "learning_rate": 1.5790417123081903e-06, | |
| "loss": 0.0135, | |
| "step": 342 | |
| }, | |
| { | |
| "clip_ratio": 0.0005754130106652156, | |
| "completion_length": 964.3333740234375, | |
| "epoch": 0.006398955272608553, | |
| "grad_norm": 0.08601634949445724, | |
| "kl": 0.012359619140625, | |
| "learning_rate": 1.5666211860780583e-06, | |
| "loss": -0.0431, | |
| "num_tokens": 3969595.0, | |
| "reward": 0.43568913638591766, | |
| "reward_std": 0.2704745829105377, | |
| "rewards/code_reward": 0.3594391196966171, | |
| "rewards/format_reward": 0.762499988079071, | |
| "step": 343 | |
| }, | |
| { | |
| "clip_ratio": 0.0005838508659508079, | |
| "epoch": 0.006417611118884381, | |
| "grad_norm": 0.08448081463575363, | |
| "kl": 0.01171875, | |
| "learning_rate": 1.5542503120528918e-06, | |
| "loss": -0.043, | |
| "step": 344 | |
| }, | |
| { | |
| "clip_ratio": 0.0007864527578931302, | |
| "epoch": 0.006436266965160207, | |
| "grad_norm": 0.08665861189365387, | |
| "kl": 0.011932373046875, | |
| "learning_rate": 1.5419296092897866e-06, | |
| "loss": -0.0432, | |
| "step": 345 | |
| }, | |
| { | |
| "clip_ratio": 0.0012224867241457105, | |
| "completion_length": 830.8750305175781, | |
| "epoch": 0.006454922811436034, | |
| "grad_norm": 0.08804712444543839, | |
| "kl": 0.03411865234375, | |
| "learning_rate": 1.529659594740755e-06, | |
| "loss": 0.0054, | |
| "num_tokens": 3999184.0, | |
| "reward": 0.12666667252779007, | |
| "reward_std": 0.1735054012387991, | |
| "rewards/code_reward": 0.0416666679084301, | |
| "rewards/format_reward": 0.8499999642372131, | |
| "step": 346 | |
| }, | |
| { | |
| "clip_ratio": 0.0012100348249077797, | |
| "epoch": 0.0064735786577118605, | |
| "grad_norm": 0.08527618646621704, | |
| "kl": 0.03369140625, | |
| "learning_rate": 1.5174407832310338e-06, | |
| "loss": 0.0053, | |
| "step": 347 | |
| }, | |
| { | |
| "clip_ratio": 0.0010465410014148802, | |
| "epoch": 0.006492234503987687, | |
| "grad_norm": 0.08590537309646606, | |
| "kl": 0.03118896484375, | |
| "learning_rate": 1.5052736874374815e-06, | |
| "loss": 0.0055, | |
| "step": 348 | |
| }, | |
| { | |
| "clip_ratio": 0.001526809181086719, | |
| "completion_length": 865.7500610351562, | |
| "epoch": 0.006510890350263514, | |
| "grad_norm": 0.09376516193151474, | |
| "kl": 0.0206298828125, | |
| "learning_rate": 1.4931588178670695e-06, | |
| "loss": 0.0089, | |
| "num_tokens": 4031674.0, | |
| "reward": 0.2808653935790062, | |
| "reward_std": 0.27811707369983196, | |
| "rewards/code_reward": 0.21794871985912323, | |
| "rewards/format_reward": 0.6291666775941849, | |
| "step": 349 | |
| }, | |
| { | |
| "clip_ratio": 0.000961648824159056, | |
| "epoch": 0.00652954619653934, | |
| "grad_norm": 0.10522089153528214, | |
| "kl": 0.02142333984375, | |
| "learning_rate": 1.4810966828354605e-06, | |
| "loss": 0.0089, | |
| "step": 350 | |
| }, | |
| { | |
| "clip_ratio": 0.0010438500612508506, | |
| "epoch": 0.006548202042815168, | |
| "grad_norm": 0.09141746908426285, | |
| "kl": 0.0203857421875, | |
| "learning_rate": 1.469087788445684e-06, | |
| "loss": 0.0084, | |
| "step": 351 | |
| }, | |
| { | |
| "clip_ratio": 0.0010631512850522995, | |
| "completion_length": 1097.8333740234375, | |
| "epoch": 0.006566857889090994, | |
| "grad_norm": 0.08537644892930984, | |
| "kl": 0.014739990234375, | |
| "learning_rate": 1.4571326385668965e-06, | |
| "loss": -0.0225, | |
| "num_tokens": 4071702.0, | |
| "reward": 0.3495173156261444, | |
| "reward_std": 0.350162148475647, | |
| "rewards/code_reward": 0.312017310410738, | |
| "rewards/format_reward": 0.3749999925494194, | |
| "step": 352 | |
| }, | |
| { | |
| "clip_ratio": 0.0007791414973326027, | |
| "epoch": 0.006585513735366821, | |
| "grad_norm": 0.08616505563259125, | |
| "kl": 0.015045166015625, | |
| "learning_rate": 1.4452317348132434e-06, | |
| "loss": -0.0226, | |
| "step": 353 | |
| }, | |
| { | |
| "clip_ratio": 0.0008178194111678749, | |
| "epoch": 0.006604169581642647, | |
| "grad_norm": 0.08682885020971298, | |
| "kl": 0.014923095703125, | |
| "learning_rate": 1.4333855765228104e-06, | |
| "loss": -0.0226, | |
| "step": 354 | |
| }, | |
| { | |
| "clip_ratio": 0.0010338413121644408, | |
| "completion_length": 677.5, | |
| "epoch": 0.006622825427918474, | |
| "grad_norm": 0.08901394158601761, | |
| "kl": 0.0438232421875, | |
| "learning_rate": 1.421594660736675e-06, | |
| "loss": 0.0347, | |
| "num_tokens": 4095762.0, | |
| "reward": 0.6164166405797005, | |
| "reward_std": 0.12990031018853188, | |
| "rewards/code_reward": 0.539749950170517, | |
| "rewards/format_reward": 0.7666666805744171, | |
| "step": 355 | |
| }, | |
| { | |
| "clip_ratio": 0.0012019489658996463, | |
| "epoch": 0.0066414812741943, | |
| "grad_norm": 0.08767423033714294, | |
| "kl": 0.03948974609375, | |
| "learning_rate": 1.4098594821780476e-06, | |
| "loss": 0.0344, | |
| "step": 356 | |
| }, | |
| { | |
| "clip_ratio": 0.001214035670273006, | |
| "epoch": 0.006660137120470127, | |
| "grad_norm": 0.09001511335372925, | |
| "kl": 0.038970947265625, | |
| "learning_rate": 1.3981805332315174e-06, | |
| "loss": 0.0346, | |
| "step": 357 | |
| }, | |
| { | |
| "clip_ratio": 0.0010124409454874694, | |
| "completion_length": 1344.5417175292969, | |
| "epoch": 0.006678792966745954, | |
| "grad_norm": 0.07684619724750519, | |
| "kl": 0.02001190185546875, | |
| "learning_rate": 1.3865583039223929e-06, | |
| "loss": 0.0339, | |
| "num_tokens": 4142671.0, | |
| "reward": 0.7462417930364609, | |
| "reward_std": 0.22497505508363247, | |
| "rewards/code_reward": 0.6674917489290237, | |
| "rewards/format_reward": 0.7874999940395355, | |
| "step": 358 | |
| }, | |
| { | |
| "clip_ratio": 0.0007824611384421587, | |
| "epoch": 0.006697448813021781, | |
| "grad_norm": 0.0674465075135231, | |
| "kl": 0.0182037353515625, | |
| "learning_rate": 1.374993281896137e-06, | |
| "loss": 0.0337, | |
| "step": 359 | |
| }, | |
| { | |
| "clip_ratio": 0.0010956324986182153, | |
| "epoch": 0.0067161046592976075, | |
| "grad_norm": 0.0694521889090538, | |
| "kl": 0.01786041259765625, | |
| "learning_rate": 1.3634859523979134e-06, | |
| "loss": 0.034, | |
| "step": 360 | |
| }, | |
| { | |
| "clip_ratio": 0.0011445771087892354, | |
| "completion_length": 637.5833740234375, | |
| "epoch": 0.006734760505573434, | |
| "grad_norm": 0.11650668829679489, | |
| "kl": 0.012542724609375, | |
| "learning_rate": 1.3520367982522208e-06, | |
| "loss": -0.0048, | |
| "num_tokens": 4168125.0, | |
| "reward": 0.3813568465411663, | |
| "reward_std": 0.24762929044663906, | |
| "rewards/code_reward": 0.29594019055366516, | |
| "rewards/format_reward": 0.8541666567325592, | |
| "step": 361 | |
| }, | |
| { | |
| "clip_ratio": 0.0014873360050842166, | |
| "epoch": 0.006753416351849261, | |
| "grad_norm": 0.11154331266880035, | |
| "kl": 0.01220703125, | |
| "learning_rate": 1.3406462998426358e-06, | |
| "loss": -0.0047, | |
| "step": 362 | |
| }, | |
| { | |
| "clip_ratio": 0.0012732939212583005, | |
| "epoch": 0.006772072198125087, | |
| "grad_norm": 0.11064095795154572, | |
| "kl": 0.01312255859375, | |
| "learning_rate": 1.3293149350916595e-06, | |
| "loss": -0.0045, | |
| "step": 363 | |
| }, | |
| { | |
| "clip_ratio": 0.0010022299247793853, | |
| "completion_length": 581.25, | |
| "epoch": 0.006790728044400914, | |
| "grad_norm": 0.20353944599628448, | |
| "kl": 0.017822265625, | |
| "learning_rate": 1.3180431794406623e-06, | |
| "loss": 0.0111, | |
| "num_tokens": 4198479.0, | |
| "reward": 0.10175329819321632, | |
| "reward_std": 0.05031761899590492, | |
| "rewards/code_reward": 0.033003296703100204, | |
| "rewards/format_reward": 0.6875, | |
| "step": 364 | |
| }, | |
| { | |
| "clip_ratio": 0.0008599580323789269, | |
| "epoch": 0.006809383890676741, | |
| "grad_norm": 0.1254553347826004, | |
| "kl": 0.0179443359375, | |
| "learning_rate": 1.3068315058299358e-06, | |
| "loss": 0.0109, | |
| "step": 365 | |
| }, | |
| { | |
| "clip_ratio": 0.0007180549946497194, | |
| "epoch": 0.006828039736952568, | |
| "grad_norm": 0.11537165939807892, | |
| "kl": 0.01995849609375, | |
| "learning_rate": 1.2956803846788503e-06, | |
| "loss": 0.011, | |
| "step": 366 | |
| }, | |
| { | |
| "clip_ratio": 0.0006652901647612453, | |
| "completion_length": 934.2917175292969, | |
| "epoch": 0.006846695583228394, | |
| "grad_norm": 0.09258420765399933, | |
| "kl": 0.011566162109375, | |
| "learning_rate": 1.284590283866116e-06, | |
| "loss": 0.0124, | |
| "num_tokens": 4241782.0, | |
| "reward": 0.21692460775375366, | |
| "reward_std": 0.24195446819067, | |
| "rewards/code_reward": 0.14484127657487988, | |
| "rewards/format_reward": 0.7208333313465118, | |
| "step": 367 | |
| }, | |
| { | |
| "clip_ratio": 0.0009541537729091942, | |
| "epoch": 0.006865351429504221, | |
| "grad_norm": 0.09379203617572784, | |
| "kl": 0.011322021484375, | |
| "learning_rate": 1.2735616687101518e-06, | |
| "loss": 0.0125, | |
| "step": 368 | |
| }, | |
| { | |
| "clip_ratio": 0.0010697944671846926, | |
| "epoch": 0.006884007275780047, | |
| "grad_norm": 0.09176654368638992, | |
| "kl": 0.01129150390625, | |
| "learning_rate": 1.2625950019495614e-06, | |
| "loss": 0.0124, | |
| "step": 369 | |
| }, | |
| { | |
| "clip_ratio": 0.000756454566726461, | |
| "completion_length": 1099.1250610351562, | |
| "epoch": 0.006902663122055874, | |
| "grad_norm": 0.07225238531827927, | |
| "kl": 0.0067138671875, | |
| "learning_rate": 1.251690743723718e-06, | |
| "loss": 0.016, | |
| "num_tokens": 4282729.0, | |
| "reward": 0.1934523843228817, | |
| "reward_std": 0.11385035887360573, | |
| "rewards/code_reward": 0.10303572192788124, | |
| "rewards/format_reward": 0.9041666388511658, | |
| "step": 370 | |
| }, | |
| { | |
| "clip_ratio": 0.000768135127145797, | |
| "epoch": 0.006921318968331701, | |
| "grad_norm": 0.0729527473449707, | |
| "kl": 0.0072021484375, | |
| "learning_rate": 1.2408493515534581e-06, | |
| "loss": 0.0163, | |
| "step": 371 | |
| }, | |
| { | |
| "clip_ratio": 0.0006400739657692611, | |
| "epoch": 0.006939974814607528, | |
| "grad_norm": 0.07251974940299988, | |
| "kl": 0.007110595703125, | |
| "learning_rate": 1.2300712803218834e-06, | |
| "loss": 0.0159, | |
| "step": 372 | |
| }, | |
| { | |
| "clip_ratio": 0.0016507117543369532, | |
| "completion_length": 431.79168701171875, | |
| "epoch": 0.0069586306608833545, | |
| "grad_norm": 0.11167676746845245, | |
| "kl": 0.014739990234375, | |
| "learning_rate": 1.2193569822552772e-06, | |
| "loss": -0.0044, | |
| "num_tokens": 4303508.0, | |
| "reward": 0.47044751420617104, | |
| "reward_std": 0.23989063128829002, | |
| "rewards/code_reward": 0.3850308656692505, | |
| "rewards/format_reward": 0.8541666567325592, | |
| "step": 373 | |
| }, | |
| { | |
| "clip_ratio": 0.001415064267348498, | |
| "epoch": 0.006977286507159181, | |
| "grad_norm": 0.10820432007312775, | |
| "kl": 0.016204833984375, | |
| "learning_rate": 1.2087069069041268e-06, | |
| "loss": -0.004, | |
| "step": 374 | |
| }, | |
| { | |
| "clip_ratio": 0.0017915331991389394, | |
| "epoch": 0.006995942353435008, | |
| "grad_norm": 0.10194279253482819, | |
| "kl": 0.01751708984375, | |
| "learning_rate": 1.1981215011242654e-06, | |
| "loss": -0.0037, | |
| "step": 375 | |
| }, | |
| { | |
| "clip_ratio": 0.0009616998431738466, | |
| "completion_length": 945.7500610351562, | |
| "epoch": 0.007014598199710834, | |
| "grad_norm": 0.06673284620046616, | |
| "kl": 0.0225830078125, | |
| "learning_rate": 1.1876012090581184e-06, | |
| "loss": -0.0215, | |
| "num_tokens": 4339982.0, | |
| "reward": 0.4850490316748619, | |
| "reward_std": 0.451216384768486, | |
| "rewards/code_reward": 0.39338236302137375, | |
| "rewards/format_reward": 0.9166666567325592, | |
| "step": 376 | |
| }, | |
| { | |
| "clip_ratio": 0.0011033907067030668, | |
| "epoch": 0.007033254045986661, | |
| "grad_norm": 0.06927090138196945, | |
| "kl": 0.02490234375, | |
| "learning_rate": 1.177146472116071e-06, | |
| "loss": -0.0215, | |
| "step": 377 | |
| }, | |
| { | |
| "clip_ratio": 0.0009509723749943078, | |
| "epoch": 0.007051909892262488, | |
| "grad_norm": 0.06608384847640991, | |
| "kl": 0.0247802734375, | |
| "learning_rate": 1.1667577289579462e-06, | |
| "loss": -0.0216, | |
| "step": 378 | |
| }, | |
| { | |
| "clip_ratio": 0.0002945388841908425, | |
| "completion_length": 615.0000152587891, | |
| "epoch": 0.007070565738538315, | |
| "grad_norm": 0.09600235521793365, | |
| "kl": 0.07568359375, | |
| "learning_rate": 1.1564354154746007e-06, | |
| "loss": 0.0006, | |
| "num_tokens": 4370618.0, | |
| "reward": 0.5112499818205833, | |
| "reward_std": 0.22198906354606152, | |
| "rewards/code_reward": 0.4166666567325592, | |
| "rewards/format_reward": 0.9458333551883698, | |
| "step": 379 | |
| }, | |
| { | |
| "clip_ratio": 0.0003342999843880534, | |
| "epoch": 0.007089221584814141, | |
| "grad_norm": 0.09851517528295517, | |
| "kl": 0.07568359375, | |
| "learning_rate": 1.146179964769635e-06, | |
| "loss": 0.0007, | |
| "step": 380 | |
| }, | |
| { | |
| "clip_ratio": 0.00041099853115156293, | |
| "epoch": 0.007107877431089968, | |
| "grad_norm": 0.10035818070173264, | |
| "kl": 0.0843505859375, | |
| "learning_rate": 1.1359918071412195e-06, | |
| "loss": 0.0008, | |
| "step": 381 | |
| }, | |
| { | |
| "clip_ratio": 0.0005624304030789062, | |
| "completion_length": 937.5417175292969, | |
| "epoch": 0.007126533277365794, | |
| "grad_norm": 0.09549058228731155, | |
| "kl": 0.0206298828125, | |
| "learning_rate": 1.1258713700640456e-06, | |
| "loss": 0.0096, | |
| "num_tokens": 4404219.0, | |
| "reward": 0.667364239692688, | |
| "reward_std": 0.4190048724412918, | |
| "rewards/code_reward": 0.576114222407341, | |
| "rewards/format_reward": 0.9124999642372131, | |
| "step": 382 | |
| }, | |
| { | |
| "clip_ratio": 0.0006415857642423362, | |
| "epoch": 0.007145189123641621, | |
| "grad_norm": 0.09549405425786972, | |
| "kl": 0.0218505859375, | |
| "learning_rate": 1.115819078171383e-06, | |
| "loss": 0.0097, | |
| "step": 383 | |
| }, | |
| { | |
| "clip_ratio": 0.0007653232023585588, | |
| "epoch": 0.0071638449699174475, | |
| "grad_norm": 0.09525733441114426, | |
| "kl": 0.021331787109375, | |
| "learning_rate": 1.1058353532372667e-06, | |
| "loss": 0.0096, | |
| "step": 384 | |
| }, | |
| { | |
| "clip_ratio": 0.000980669748969376, | |
| "completion_length": 1172.1666870117188, | |
| "epoch": 0.007182500816193275, | |
| "grad_norm": 0.07197841256856918, | |
| "kl": 0.01171875, | |
| "learning_rate": 1.0959206141587998e-06, | |
| "loss": -0.0047, | |
| "num_tokens": 4448983.0, | |
| "reward": 0.11828883737325668, | |
| "reward_std": 0.10133323073387146, | |
| "rewards/code_reward": 0.05703882873058319, | |
| "rewards/format_reward": 0.6124999821186066, | |
| "step": 385 | |
| }, | |
| { | |
| "clip_ratio": 0.0007983763935044408, | |
| "epoch": 0.0072011566624691015, | |
| "grad_norm": 0.07335319370031357, | |
| "kl": 0.01263427734375, | |
| "learning_rate": 1.0860752769385766e-06, | |
| "loss": -0.0049, | |
| "step": 386 | |
| }, | |
| { | |
| "clip_ratio": 0.001325059012742713, | |
| "epoch": 0.007219812508744928, | |
| "grad_norm": 0.07985582202672958, | |
| "kl": 0.011871337890625, | |
| "learning_rate": 1.0762997546672279e-06, | |
| "loss": -0.005, | |
| "step": 387 | |
| }, | |
| { | |
| "clip_ratio": 0.0009500840096734464, | |
| "completion_length": 764.8750305175781, | |
| "epoch": 0.007238468355020755, | |
| "grad_norm": 0.103624626994133, | |
| "kl": 0.019287109375, | |
| "learning_rate": 1.0665944575060914e-06, | |
| "loss": 0.0971, | |
| "num_tokens": 4478560.0, | |
| "reward": 0.47962963581085205, | |
| "reward_std": 0.2753693163394928, | |
| "rewards/code_reward": 0.383796289563179, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 388 | |
| }, | |
| { | |
| "clip_ratio": 0.0007495690369978547, | |
| "epoch": 0.007257124201296581, | |
| "grad_norm": 0.1009078398346901, | |
| "kl": 0.02117919921875, | |
| "learning_rate": 1.056959792669997e-06, | |
| "loss": 0.0968, | |
| "step": 389 | |
| }, | |
| { | |
| "clip_ratio": 0.0008809762366581708, | |
| "epoch": 0.007275780047572408, | |
| "grad_norm": 0.10338608175516129, | |
| "kl": 0.0205078125, | |
| "learning_rate": 1.0473961644101856e-06, | |
| "loss": 0.0969, | |
| "step": 390 | |
| }, | |
| { | |
| "clip_ratio": 0.0008085968729574233, | |
| "completion_length": 664.7916870117188, | |
| "epoch": 0.007294435893848234, | |
| "grad_norm": 0.09412232786417007, | |
| "kl": 0.02545166015625, | |
| "learning_rate": 1.037903973997345e-06, | |
| "loss": 0.0, | |
| "num_tokens": 4507655.0, | |
| "reward": 0.5595833584666252, | |
| "reward_std": 0.18141867592930794, | |
| "rewards/code_reward": 0.4583333432674408, | |
| "rewards/format_reward": 1.012499988079071, | |
| "step": 391 | |
| }, | |
| { | |
| "clip_ratio": 0.0006834749365225434, | |
| "epoch": 0.007313091740124062, | |
| "grad_norm": 0.13709917664527893, | |
| "kl": 0.0255126953125, | |
| "learning_rate": 1.0284836197047737e-06, | |
| "loss": 0.0001, | |
| "step": 392 | |
| }, | |
| { | |
| "clip_ratio": 0.0006806632154621184, | |
| "epoch": 0.007331747586399888, | |
| "grad_norm": 0.09327536076307297, | |
| "kl": 0.0245361328125, | |
| "learning_rate": 1.0191354967916712e-06, | |
| "loss": 0.0001, | |
| "step": 393 | |
| }, | |
| { | |
| "clip_ratio": 0.0008892881742212921, | |
| "completion_length": 666.8333435058594, | |
| "epoch": 0.007350403432675715, | |
| "grad_norm": 0.09906277060508728, | |
| "kl": 0.02410888671875, | |
| "learning_rate": 1.0098599974865515e-06, | |
| "loss": 0.0111, | |
| "num_tokens": 4542367.0, | |
| "reward": 0.771384060382843, | |
| "reward_std": 0.17679990455508232, | |
| "rewards/code_reward": 0.6901340931653976, | |
| "rewards/format_reward": 0.8125, | |
| "step": 394 | |
| }, | |
| { | |
| "clip_ratio": 0.0008279377470898908, | |
| "epoch": 0.007369059278951541, | |
| "grad_norm": 0.10521873086690903, | |
| "kl": 0.02398681640625, | |
| "learning_rate": 1.0006575109707898e-06, | |
| "loss": 0.0108, | |
| "step": 395 | |
| }, | |
| { | |
| "clip_ratio": 0.0009161636990029365, | |
| "epoch": 0.007387715125227368, | |
| "grad_norm": 0.09651351720094681, | |
| "kl": 0.023193359375, | |
| "learning_rate": 9.915284233622877e-07, | |
| "loss": 0.011, | |
| "step": 396 | |
| }, | |
| { | |
| "clip_ratio": 0.00039033169741742313, | |
| "completion_length": 786.7500457763672, | |
| "epoch": 0.0074063709715031945, | |
| "grad_norm": 0.09486052393913269, | |
| "kl": 0.02703857421875, | |
| "learning_rate": 9.824731176992796e-07, | |
| "loss": 0.0268, | |
| "num_tokens": 4569841.0, | |
| "reward": 0.4910014681518078, | |
| "reward_std": 0.2589704990386963, | |
| "rewards/code_reward": 0.37975146202370524, | |
| "rewards/format_reward": 1.1124999523162842, | |
| "step": 397 | |
| }, | |
| { | |
| "clip_ratio": 0.00043224109685979784, | |
| "epoch": 0.007425026817779021, | |
| "grad_norm": 0.09329359233379364, | |
| "kl": 0.03033447265625, | |
| "learning_rate": 9.734919739242543e-07, | |
| "loss": 0.0269, | |
| "step": 398 | |
| }, | |
| { | |
| "clip_ratio": 0.0005088059406261891, | |
| "epoch": 0.0074436826640548485, | |
| "grad_norm": 0.09744512289762497, | |
| "kl": 0.0316162109375, | |
| "learning_rate": 9.645853688680177e-07, | |
| "loss": 0.027, | |
| "step": 399 | |
| }, | |
| { | |
| "clip_ratio": 0.0007269750349223614, | |
| "completion_length": 853.0833435058594, | |
| "epoch": 0.007462338510330675, | |
| "grad_norm": 0.07783529907464981, | |
| "kl": 0.029083251953125, | |
| "learning_rate": 9.557536762338786e-07, | |
| "loss": 0.0011, | |
| "num_tokens": 4600515.0, | |
| "reward": 0.4749346524477005, | |
| "reward_std": 0.22702517919242382, | |
| "rewards/code_reward": 0.39910128712654114, | |
| "rewards/format_reward": 0.7583333253860474, | |
| "step": 400 | |
| }, | |
| { | |
| "clip_ratio": 0.0009095004643313587, | |
| "epoch": 0.007480994356606502, | |
| "grad_norm": 0.07656848430633545, | |
| "kl": 0.029296875, | |
| "learning_rate": 9.46997266581973e-07, | |
| "loss": 0.0013, | |
| "step": 401 | |
| }, | |
| { | |
| "clip_ratio": 0.0006659157224930823, | |
| "epoch": 0.007499650202882328, | |
| "grad_norm": 0.07752339541912079, | |
| "kl": 0.03289794921875, | |
| "learning_rate": 9.383165073137115e-07, | |
| "loss": 0.001, | |
| "step": 402 | |
| }, | |
| { | |
| "clip_ratio": 0.0015702053788118064, | |
| "completion_length": 578.1666717529297, | |
| "epoch": 0.007518306049158155, | |
| "grad_norm": 0.11208923161029816, | |
| "kl": 0.0478515625, | |
| "learning_rate": 9.297117626563687e-07, | |
| "loss": -0.0263, | |
| "num_tokens": 4624963.0, | |
| "reward": 0.8968709111213684, | |
| "reward_std": 0.3033272698521614, | |
| "rewards/code_reward": 0.78228759765625, | |
| "rewards/format_reward": 1.1458333134651184, | |
| "step": 403 | |
| }, | |
| { | |
| "clip_ratio": 0.0014707182417623699, | |
| "epoch": 0.007536961895433981, | |
| "grad_norm": 0.11273815482854843, | |
| "kl": 0.048583984375, | |
| "learning_rate": 9.211833936477957e-07, | |
| "loss": -0.0262, | |
| "step": 404 | |
| }, | |
| { | |
| "clip_ratio": 0.0014846965787000954, | |
| "epoch": 0.007555617741709809, | |
| "grad_norm": 0.11112023890018463, | |
| "kl": 0.0447998046875, | |
| "learning_rate": 9.127317581212753e-07, | |
| "loss": -0.0261, | |
| "step": 405 | |
| }, | |
| { | |
| "clip_ratio": 0.000766730576287955, | |
| "completion_length": 860.0000305175781, | |
| "epoch": 0.007574273587985635, | |
| "grad_norm": 0.07397118955850601, | |
| "kl": 0.03369140625, | |
| "learning_rate": 9.043572106905084e-07, | |
| "loss": 0.0091, | |
| "num_tokens": 4658083.0, | |
| "reward": 0.6170833520591259, | |
| "reward_std": 0.16897812485694885, | |
| "rewards/code_reward": 0.5416666567325592, | |
| "rewards/format_reward": 0.7541666999459267, | |
| "step": 406 | |
| }, | |
| { | |
| "clip_ratio": 0.0006420460704248399, | |
| "epoch": 0.007592929434261462, | |
| "grad_norm": 0.0758887380361557, | |
| "kl": 0.03009033203125, | |
| "learning_rate": 8.960601027347321e-07, | |
| "loss": 0.0092, | |
| "step": 407 | |
| }, | |
| { | |
| "clip_ratio": 0.0007185917929746211, | |
| "epoch": 0.007611585280537288, | |
| "grad_norm": 0.07315798103809357, | |
| "kl": 0.0299072265625, | |
| "learning_rate": 8.878407823839788e-07, | |
| "loss": 0.0091, | |
| "step": 408 | |
| }, | |
| { | |
| "clip_ratio": 0.0008533093787264079, | |
| "completion_length": 1027.0000305175781, | |
| "epoch": 0.007630241126813115, | |
| "grad_norm": 0.08570308238267899, | |
| "kl": 0.03460693359375, | |
| "learning_rate": 8.796995945044689e-07, | |
| "loss": 0.0127, | |
| "num_tokens": 4699639.0, | |
| "reward": 0.06000000238418579, | |
| "reward_std": 0.05202655866742134, | |
| "rewards/code_reward": 0.0, | |
| "rewards/format_reward": 0.5999999940395355, | |
| "step": 409 | |
| }, | |
| { | |
| "clip_ratio": 0.0009294236078858376, | |
| "epoch": 0.0076488969730889415, | |
| "grad_norm": 0.07877162843942642, | |
| "kl": 0.0330810546875, | |
| "learning_rate": 8.716368806841405e-07, | |
| "loss": 0.0129, | |
| "step": 410 | |
| }, | |
| { | |
| "clip_ratio": 0.0008056100341491401, | |
| "epoch": 0.007667552819364768, | |
| "grad_norm": 0.07954779267311096, | |
| "kl": 0.031005859375, | |
| "learning_rate": 8.636529792183171e-07, | |
| "loss": 0.0125, | |
| "step": 411 | |
| }, | |
| { | |
| "clip_ratio": 0.0007956986082717776, | |
| "completion_length": 737.4166870117188, | |
| "epoch": 0.0076862086656405955, | |
| "grad_norm": 0.08732157200574875, | |
| "kl": 0.027008056640625, | |
| "learning_rate": 8.557482250955144e-07, | |
| "loss": 0.0307, | |
| "num_tokens": 4734041.0, | |
| "reward": 0.7656373083591461, | |
| "reward_std": 0.2801913693547249, | |
| "rewards/code_reward": 0.6764705926179886, | |
| "rewards/format_reward": 0.8916666507720947, | |
| "step": 412 | |
| }, | |
| { | |
| "clip_ratio": 0.0005478582752402872, | |
| "epoch": 0.007704864511916422, | |
| "grad_norm": 0.0853348970413208, | |
| "kl": 0.027374267578125, | |
| "learning_rate": 8.479229499833844e-07, | |
| "loss": 0.0308, | |
| "step": 413 | |
| }, | |
| { | |
| "clip_ratio": 0.0008438964432571083, | |
| "epoch": 0.007723520358192249, | |
| "grad_norm": 0.08625921607017517, | |
| "kl": 0.02655029296875, | |
| "learning_rate": 8.401774822147976e-07, | |
| "loss": 0.0306, | |
| "step": 414 | |
| }, | |
| { | |
| "clip_ratio": 0.0019030745315831155, | |
| "completion_length": 749.3750152587891, | |
| "epoch": 0.007742176204468075, | |
| "grad_norm": 0.09860255569219589, | |
| "kl": 0.041900634765625, | |
| "learning_rate": 8.325121467740695e-07, | |
| "loss": -0.0314, | |
| "num_tokens": 4770374.0, | |
| "reward": 0.16708937659859657, | |
| "reward_std": 0.18778712674975395, | |
| "rewards/code_reward": 0.11292270570993423, | |
| "rewards/format_reward": 0.5416666716337204, | |
| "step": 415 | |
| }, | |
| { | |
| "clip_ratio": 0.0017334477743133903, | |
| "epoch": 0.007760832050743902, | |
| "grad_norm": 0.09663218259811401, | |
| "kl": 0.039215087890625, | |
| "learning_rate": 8.249272652833226e-07, | |
| "loss": -0.0314, | |
| "step": 416 | |
| }, | |
| { | |
| "clip_ratio": 0.0020351834245957434, | |
| "epoch": 0.007779487897019728, | |
| "grad_norm": 0.10066119581460953, | |
| "kl": 0.0433349609375, | |
| "learning_rate": 8.174231559889931e-07, | |
| "loss": -0.0313, | |
| "step": 417 | |
| }, | |
| { | |
| "clip_ratio": 0.0017151011270470917, | |
| "completion_length": 631.6666870117188, | |
| "epoch": 0.007798143743295555, | |
| "grad_norm": 0.09030763059854507, | |
| "kl": 0.046661376953125, | |
| "learning_rate": 8.100001337484787e-07, | |
| "loss": 0.0406, | |
| "num_tokens": 4798446.0, | |
| "reward": 0.9333169162273407, | |
| "reward_std": 0.1303430050611496, | |
| "rewards/code_reward": 0.8399835228919983, | |
| "rewards/format_reward": 0.9333333075046539, | |
| "step": 418 | |
| }, | |
| { | |
| "clip_ratio": 0.0014318364555947483, | |
| "epoch": 0.007816799589571382, | |
| "grad_norm": 0.1113353744149208, | |
| "kl": 0.051666259765625, | |
| "learning_rate": 8.026585100169251e-07, | |
| "loss": 0.0409, | |
| "step": 419 | |
| }, | |
| { | |
| "clip_ratio": 0.0014859928633086383, | |
| "epoch": 0.007835455435847209, | |
| "grad_norm": 0.09183201938867569, | |
| "kl": 0.055267333984375, | |
| "learning_rate": 7.953985928341601e-07, | |
| "loss": 0.0409, | |
| "step": 420 | |
| }, | |
| { | |
| "clip_ratio": 0.0009874806564766914, | |
| "completion_length": 722.5833435058594, | |
| "epoch": 0.007854111282123035, | |
| "grad_norm": 0.301491379737854, | |
| "kl": 0.03594970703125, | |
| "learning_rate": 7.882206868117693e-07, | |
| "loss": 0.0123, | |
| "num_tokens": 4831364.0, | |
| "reward": 0.46648313850164413, | |
| "reward_std": 0.24989983439445496, | |
| "rewards/code_reward": 0.39314980432391167, | |
| "rewards/format_reward": 0.7333333492279053, | |
| "step": 421 | |
| }, | |
| { | |
| "clip_ratio": 0.0007859274046495557, | |
| "epoch": 0.007872767128398862, | |
| "grad_norm": 0.09945830702781677, | |
| "kl": 0.03179931640625, | |
| "learning_rate": 7.81125093120313e-07, | |
| "loss": 0.0124, | |
| "step": 422 | |
| }, | |
| { | |
| "clip_ratio": 0.0010357568971812725, | |
| "epoch": 0.007891422974674689, | |
| "grad_norm": 0.09847865253686905, | |
| "kl": 0.03509521484375, | |
| "learning_rate": 7.741121094766916e-07, | |
| "loss": 0.0124, | |
| "step": 423 | |
| }, | |
| { | |
| "clip_ratio": 0.0015808032476343215, | |
| "completion_length": 677.6250152587891, | |
| "epoch": 0.007910078820950515, | |
| "grad_norm": 0.1073075532913208, | |
| "kl": 0.05718994140625, | |
| "learning_rate": 7.671820301316532e-07, | |
| "loss": 0.0359, | |
| "num_tokens": 4852811.0, | |
| "reward": 0.2411028817296028, | |
| "reward_std": 0.08415344171226025, | |
| "rewards/code_reward": 0.15360287349903956, | |
| "rewards/format_reward": 0.875, | |
| "step": 424 | |
| }, | |
| { | |
| "clip_ratio": 0.0017813361482694745, | |
| "epoch": 0.007928734667226342, | |
| "grad_norm": 0.10579974949359894, | |
| "kl": 0.05438232421875, | |
| "learning_rate": 7.603351458574474e-07, | |
| "loss": 0.0363, | |
| "step": 425 | |
| }, | |
| { | |
| "clip_ratio": 0.001651192782446742, | |
| "epoch": 0.007947390513502168, | |
| "grad_norm": 0.11324102431535721, | |
| "kl": 0.05828857421875, | |
| "learning_rate": 7.535717439356255e-07, | |
| "loss": 0.0361, | |
| "step": 426 | |
| }, | |
| { | |
| "clip_ratio": 0.0014180371072143316, | |
| "completion_length": 693.3333435058594, | |
| "epoch": 0.007966046359777995, | |
| "grad_norm": 0.16692698001861572, | |
| "kl": 0.0213623046875, | |
| "learning_rate": 7.46892108144986e-07, | |
| "loss": 0.0654, | |
| "num_tokens": 4883755.0, | |
| "reward": 0.6448469460010529, | |
| "reward_std": 0.49255189299583435, | |
| "rewards/code_reward": 0.5340135842561722, | |
| "rewards/format_reward": 1.1083332896232605, | |
| "step": 427 | |
| }, | |
| { | |
| "clip_ratio": 0.0013092211447656155, | |
| "epoch": 0.007984702206053821, | |
| "grad_norm": 0.09344299882650375, | |
| "kl": 0.02069091796875, | |
| "learning_rate": 7.402965187496697e-07, | |
| "loss": 0.0651, | |
| "step": 428 | |
| }, | |
| { | |
| "clip_ratio": 0.0009931237436830997, | |
| "epoch": 0.00800335805232965, | |
| "grad_norm": 0.0950753316283226, | |
| "kl": 0.02081298828125, | |
| "learning_rate": 7.337852524873974e-07, | |
| "loss": 0.0652, | |
| "step": 429 | |
| }, | |
| { | |
| "clip_ratio": 0.000936219235882163, | |
| "completion_length": 608.0000152587891, | |
| "epoch": 0.008022013898605476, | |
| "grad_norm": 0.10651998966932297, | |
| "kl": 0.080657958984375, | |
| "learning_rate": 7.273585825578608e-07, | |
| "loss": -0.0054, | |
| "num_tokens": 4910095.0, | |
| "reward": 0.6445792466402054, | |
| "reward_std": 0.16696248203516006, | |
| "rewards/code_reward": 0.5420792102813721, | |
| "rewards/format_reward": 1.024999976158142, | |
| "step": 430 | |
| }, | |
| { | |
| "clip_ratio": 0.0005899647367186844, | |
| "epoch": 0.008040669744881303, | |
| "grad_norm": 0.10619425773620605, | |
| "kl": 0.083984375, | |
| "learning_rate": 7.21016778611259e-07, | |
| "loss": -0.0052, | |
| "step": 431 | |
| }, | |
| { | |
| "clip_ratio": 0.0008555436506867409, | |
| "epoch": 0.00805932559115713, | |
| "grad_norm": 0.10411829501390457, | |
| "kl": 0.0823974609375, | |
| "learning_rate": 7.147601067369835e-07, | |
| "loss": -0.0054, | |
| "step": 432 | |
| }, | |
| { | |
| "clip_ratio": 0.0007757125422358513, | |
| "completion_length": 393.5833435058594, | |
| "epoch": 0.008077981437432956, | |
| "grad_norm": 0.11761093884706497, | |
| "kl": 0.08935546875, | |
| "learning_rate": 7.085888294524561e-07, | |
| "loss": -0.0138, | |
| "num_tokens": 4927329.0, | |
| "reward": 0.6537500023841858, | |
| "reward_std": 0.17917166277766228, | |
| "rewards/code_reward": 0.5416666679084301, | |
| "rewards/format_reward": 1.120833307504654, | |
| "step": 433 | |
| }, | |
| { | |
| "clip_ratio": 0.0007667961181141436, | |
| "epoch": 0.008096637283708782, | |
| "grad_norm": 0.11278140544891357, | |
| "kl": 0.076904296875, | |
| "learning_rate": 7.025032056921117e-07, | |
| "loss": -0.0137, | |
| "step": 434 | |
| }, | |
| { | |
| "clip_ratio": 0.000893244257895276, | |
| "epoch": 0.008115293129984609, | |
| "grad_norm": 0.1142934262752533, | |
| "kl": 0.077880859375, | |
| "learning_rate": 6.965034907965349e-07, | |
| "loss": -0.0139, | |
| "step": 435 | |
| }, | |
| { | |
| "clip_ratio": 0.0005022610421292484, | |
| "completion_length": 807.9583435058594, | |
| "epoch": 0.008133948976260436, | |
| "grad_norm": 0.0828719511628151, | |
| "kl": 0.0184326171875, | |
| "learning_rate": 6.905899365017462e-07, | |
| "loss": 0.027, | |
| "num_tokens": 4961552.0, | |
| "reward": 0.7701181471347809, | |
| "reward_std": 0.3888111412525177, | |
| "rewards/code_reward": 0.6847014874219894, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 436 | |
| }, | |
| { | |
| "clip_ratio": 0.0007263242732733488, | |
| "epoch": 0.008152604822536262, | |
| "grad_norm": 0.08436731994152069, | |
| "kl": 0.0183563232421875, | |
| "learning_rate": 6.847627909286409e-07, | |
| "loss": 0.0272, | |
| "step": 437 | |
| }, | |
| { | |
| "clip_ratio": 0.0007911297434475273, | |
| "epoch": 0.008171260668812089, | |
| "grad_norm": 0.08241833746433258, | |
| "kl": 0.0186309814453125, | |
| "learning_rate": 6.790222985725761e-07, | |
| "loss": 0.027, | |
| "step": 438 | |
| }, | |
| { | |
| "clip_ratio": 0.0008176612027455121, | |
| "completion_length": 741.7083435058594, | |
| "epoch": 0.008189916515087915, | |
| "grad_norm": 0.09797824174165726, | |
| "kl": 0.02587890625, | |
| "learning_rate": 6.733687002931141e-07, | |
| "loss": -0.0261, | |
| "num_tokens": 4989097.0, | |
| "reward": 0.3370833322405815, | |
| "reward_std": 0.2753293476998806, | |
| "rewards/code_reward": 0.2291666716337204, | |
| "rewards/format_reward": 1.0791666209697723, | |
| "step": 439 | |
| }, | |
| { | |
| "clip_ratio": 0.0010906177922151983, | |
| "epoch": 0.008208572361363742, | |
| "grad_norm": 0.10210465639829636, | |
| "kl": 0.026519775390625, | |
| "learning_rate": 6.678022333039158e-07, | |
| "loss": -0.0261, | |
| "step": 440 | |
| }, | |
| { | |
| "clip_ratio": 0.0006749743188265711, | |
| "epoch": 0.008227228207639568, | |
| "grad_norm": 0.09442983567714691, | |
| "kl": 0.027008056640625, | |
| "learning_rate": 6.623231311627876e-07, | |
| "loss": -0.0263, | |
| "step": 441 | |
| }, | |
| { | |
| "clip_ratio": 0.0011484751594252884, | |
| "completion_length": 866.75, | |
| "epoch": 0.008245884053915395, | |
| "grad_norm": 0.07345132529735565, | |
| "kl": 0.03179931640625, | |
| "learning_rate": 6.569316237618811e-07, | |
| "loss": 0.0639, | |
| "num_tokens": 5026687.0, | |
| "reward": 0.2147694230079651, | |
| "reward_std": 0.33931225538253784, | |
| "rewards/code_reward": 0.132686085999012, | |
| "rewards/format_reward": 0.8208333253860474, | |
| "step": 442 | |
| }, | |
| { | |
| "clip_ratio": 0.0010752396774478257, | |
| "epoch": 0.008264539900191223, | |
| "grad_norm": 0.07625681161880493, | |
| "kl": 0.0343017578125, | |
| "learning_rate": 6.516279373180499e-07, | |
| "loss": 0.0637, | |
| "step": 443 | |
| }, | |
| { | |
| "clip_ratio": 0.001316602574661374, | |
| "epoch": 0.00828319574646705, | |
| "grad_norm": 0.07411348074674606, | |
| "kl": 0.0333251953125, | |
| "learning_rate": 6.464122943633543e-07, | |
| "loss": 0.0638, | |
| "step": 444 | |
| }, | |
| { | |
| "clip_ratio": 0.0007968830032041296, | |
| "completion_length": 739.5000305175781, | |
| "epoch": 0.008301851592742876, | |
| "grad_norm": 0.09090346097946167, | |
| "kl": 0.0362548828125, | |
| "learning_rate": 6.412849137357271e-07, | |
| "loss": 0.0124, | |
| "num_tokens": 5059039.0, | |
| "reward": 0.6221906468272209, | |
| "reward_std": 0.09545084834098816, | |
| "rewards/code_reward": 0.530940592288971, | |
| "rewards/format_reward": 0.9124999940395355, | |
| "step": 445 | |
| }, | |
| { | |
| "clip_ratio": 0.0005075681256130338, | |
| "epoch": 0.008320507439018703, | |
| "grad_norm": 0.09324011206626892, | |
| "kl": 0.0374755859375, | |
| "learning_rate": 6.3624601056979e-07, | |
| "loss": 0.0122, | |
| "step": 446 | |
| }, | |
| { | |
| "clip_ratio": 0.0002910211042035371, | |
| "epoch": 0.00833916328529453, | |
| "grad_norm": 0.0913727805018425, | |
| "kl": 0.0367431640625, | |
| "learning_rate": 6.312957962878278e-07, | |
| "loss": 0.0122, | |
| "step": 447 | |
| }, | |
| { | |
| "clip_ratio": 0.0009194318408844993, | |
| "completion_length": 722.7500305175781, | |
| "epoch": 0.008357819131570356, | |
| "grad_norm": 0.08921171724796295, | |
| "kl": 0.015167236328125, | |
| "learning_rate": 6.264344785909181e-07, | |
| "loss": -0.0187, | |
| "num_tokens": 5088301.0, | |
| "reward": 0.3760389983654022, | |
| "reward_std": 0.21540092676877975, | |
| "rewards/code_reward": 0.25312230736017227, | |
| "rewards/format_reward": 1.2291666865348816, | |
| "step": 448 | |
| }, | |
| { | |
| "clip_ratio": 0.000731065621948801, | |
| "epoch": 0.008376474977846183, | |
| "grad_norm": 0.08806627243757248, | |
| "kl": 0.0145263671875, | |
| "learning_rate": 6.216622614502149e-07, | |
| "loss": -0.0183, | |
| "step": 449 | |
| }, | |
| { | |
| "clip_ratio": 0.001043492229655385, | |
| "epoch": 0.008395130824122009, | |
| "grad_norm": 0.08825229853391647, | |
| "kl": 0.013092041015625, | |
| "learning_rate": 6.169793450983916e-07, | |
| "loss": -0.0185, | |
| "step": 450 | |
| }, | |
| { | |
| "clip_ratio": 0.0011616212141234428, | |
| "completion_length": 723.0416870117188, | |
| "epoch": 0.008413786670397836, | |
| "grad_norm": 0.10913647711277008, | |
| "kl": 0.0169677734375, | |
| "learning_rate": 6.123859260212393e-07, | |
| "loss": 0.0171, | |
| "num_tokens": 5114834.0, | |
| "reward": 0.4591666907072067, | |
| "reward_std": 0.2421144973486662, | |
| "rewards/code_reward": 0.375, | |
| "rewards/format_reward": 0.8416666686534882, | |
| "step": 451 | |
| }, | |
| { | |
| "clip_ratio": 0.0010601775429677218, | |
| "epoch": 0.008432442516673662, | |
| "grad_norm": 0.1069192886352539, | |
| "kl": 0.0181884765625, | |
| "learning_rate": 6.07882196949423e-07, | |
| "loss": 0.0172, | |
| "step": 452 | |
| }, | |
| { | |
| "clip_ratio": 0.0011184008326381445, | |
| "epoch": 0.008451098362949489, | |
| "grad_norm": 0.10386621952056885, | |
| "kl": 0.017730712890625, | |
| "learning_rate": 6.034683468503948e-07, | |
| "loss": 0.0172, | |
| "step": 453 | |
| }, | |
| { | |
| "clip_ratio": 0.001289009174797684, | |
| "completion_length": 979.8333740234375, | |
| "epoch": 0.008469754209225315, | |
| "grad_norm": 0.08415919542312622, | |
| "kl": 0.011962890625, | |
| "learning_rate": 5.991445609204641e-07, | |
| "loss": -0.0303, | |
| "num_tokens": 5149462.0, | |
| "reward": 0.5253205597400665, | |
| "reward_std": 0.1456993781030178, | |
| "rewards/code_reward": 0.4149038642644882, | |
| "rewards/format_reward": 1.1041666269302368, | |
| "step": 454 | |
| }, | |
| { | |
| "clip_ratio": 0.000828628777526319, | |
| "epoch": 0.008488410055501142, | |
| "grad_norm": 0.0916297659277916, | |
| "kl": 0.011932373046875, | |
| "learning_rate": 5.949110205770292e-07, | |
| "loss": -0.0305, | |
| "step": 455 | |
| }, | |
| { | |
| "clip_ratio": 0.0010398888844065368, | |
| "epoch": 0.00850706590177697, | |
| "grad_norm": 0.08343882113695145, | |
| "kl": 0.0126953125, | |
| "learning_rate": 5.90767903450964e-07, | |
| "loss": -0.0306, | |
| "step": 456 | |
| }, | |
| { | |
| "clip_ratio": 0.0010439236648380756, | |
| "completion_length": 754.125, | |
| "epoch": 0.008525721748052797, | |
| "grad_norm": 0.08621830493211746, | |
| "kl": 0.019927978515625, | |
| "learning_rate": 5.867153833791652e-07, | |
| "loss": 0.0192, | |
| "num_tokens": 5181685.0, | |
| "reward": 0.3584918677806854, | |
| "reward_std": 0.23101448267698288, | |
| "rewards/code_reward": 0.2543252221075818, | |
| "rewards/format_reward": 1.0416666567325592, | |
| "step": 457 | |
| }, | |
| { | |
| "clip_ratio": 0.0011347323888912797, | |
| "epoch": 0.008544377594328623, | |
| "grad_norm": 0.08586623519659042, | |
| "kl": 0.020263671875, | |
| "learning_rate": 5.827536303972587e-07, | |
| "loss": 0.0193, | |
| "step": 458 | |
| }, | |
| { | |
| "clip_ratio": 0.0011065362195950001, | |
| "epoch": 0.00856303344060445, | |
| "grad_norm": 0.08708977699279785, | |
| "kl": 0.018890380859375, | |
| "learning_rate": 5.78882810732465e-07, | |
| "loss": 0.0192, | |
| "step": 459 | |
| }, | |
| { | |
| "clip_ratio": 0.0007787059585098177, | |
| "completion_length": 975.9167175292969, | |
| "epoch": 0.008581689286880276, | |
| "grad_norm": 0.11942039430141449, | |
| "kl": 0.025390625, | |
| "learning_rate": 5.75103086796625e-07, | |
| "loss": 0.0441, | |
| "num_tokens": 5223179.0, | |
| "reward": 0.0820833370089531, | |
| "reward_std": 0.06146477162837982, | |
| "rewards/code_reward": 0.0, | |
| "rewards/format_reward": 0.8208333253860474, | |
| "step": 460 | |
| }, | |
| { | |
| "clip_ratio": 0.0007732175581622869, | |
| "epoch": 0.008600345133156103, | |
| "grad_norm": 0.0921056792140007, | |
| "kl": 0.02691650390625, | |
| "learning_rate": 5.714146171793846e-07, | |
| "loss": 0.0444, | |
| "step": 461 | |
| }, | |
| { | |
| "clip_ratio": 0.0012547720107249916, | |
| "epoch": 0.00861900097943193, | |
| "grad_norm": 0.09074374288320541, | |
| "kl": 0.02606201171875, | |
| "learning_rate": 5.678175566415422e-07, | |
| "loss": 0.0445, | |
| "step": 462 | |
| }, | |
| { | |
| "clip_ratio": 0.0012275170302018523, | |
| "completion_length": 753.1666870117188, | |
| "epoch": 0.008637656825707756, | |
| "grad_norm": 0.10298697650432587, | |
| "kl": 0.0291748046875, | |
| "learning_rate": 5.643120561085528e-07, | |
| "loss": -0.0121, | |
| "num_tokens": 5257791.0, | |
| "reward": 0.3049456551671028, | |
| "reward_std": 0.2707311660051346, | |
| "rewards/code_reward": 0.22036230191588402, | |
| "rewards/format_reward": 0.8458333015441895, | |
| "step": 463 | |
| }, | |
| { | |
| "clip_ratio": 0.0011390139115974307, | |
| "epoch": 0.008656312671983583, | |
| "grad_norm": 0.10827159136533737, | |
| "kl": 0.02630615234375, | |
| "learning_rate": 5.608982626641991e-07, | |
| "loss": -0.0124, | |
| "step": 464 | |
| }, | |
| { | |
| "clip_ratio": 0.001239880220964551, | |
| "epoch": 0.00867496851825941, | |
| "grad_norm": 0.10058361291885376, | |
| "kl": 0.0302734375, | |
| "learning_rate": 5.575763195444166e-07, | |
| "loss": -0.0123, | |
| "step": 465 | |
| }, | |
| { | |
| "clip_ratio": 0.0006581109191756696, | |
| "completion_length": 424.9583435058594, | |
| "epoch": 0.008693624364535236, | |
| "grad_norm": 0.1420270800590515, | |
| "kl": 0.0567169189453125, | |
| "learning_rate": 5.543463661312847e-07, | |
| "loss": -0.0107, | |
| "num_tokens": 5276846.0, | |
| "reward": 0.6923423409461975, | |
| "reward_std": 0.068264564499259, | |
| "rewards/code_reward": 0.5506756789982319, | |
| "rewards/format_reward": 1.4166666269302368, | |
| "step": 466 | |
| }, | |
| { | |
| "clip_ratio": 0.0005906374426558614, | |
| "epoch": 0.008712280210811062, | |
| "grad_norm": 0.13097578287124634, | |
| "kl": 0.0464019775390625, | |
| "learning_rate": 5.512085379471808e-07, | |
| "loss": -0.0104, | |
| "step": 467 | |
| }, | |
| { | |
| "clip_ratio": 0.0008538902620784938, | |
| "epoch": 0.008730936057086889, | |
| "grad_norm": 0.13688351213932037, | |
| "kl": 0.0464019775390625, | |
| "learning_rate": 5.481629666490903e-07, | |
| "loss": -0.0102, | |
| "step": 468 | |
| }, | |
| { | |
| "clip_ratio": 0.0010946058901026845, | |
| "completion_length": 817.5833435058594, | |
| "epoch": 0.008749591903362716, | |
| "grad_norm": 0.09143988788127899, | |
| "kl": 0.03515625, | |
| "learning_rate": 5.452097800230853e-07, | |
| "loss": -0.0014, | |
| "num_tokens": 5311588.0, | |
| "reward": 0.29928937181830406, | |
| "reward_std": 0.14651047810912132, | |
| "rewards/code_reward": 0.21553937532007694, | |
| "rewards/format_reward": 0.8374999761581421, | |
| "step": 469 | |
| }, | |
| { | |
| "clip_ratio": 0.0012111273827031255, | |
| "epoch": 0.008768247749638544, | |
| "grad_norm": 0.10869612544775009, | |
| "kl": 0.03814697265625, | |
| "learning_rate": 5.423491019789623e-07, | |
| "loss": -0.0015, | |
| "step": 470 | |
| }, | |
| { | |
| "clip_ratio": 0.0006720305973431095, | |
| "epoch": 0.00878690359591437, | |
| "grad_norm": 0.10290330648422241, | |
| "kl": 0.03741455078125, | |
| "learning_rate": 5.395810525450425e-07, | |
| "loss": -0.0014, | |
| "step": 471 | |
| }, | |
| { | |
| "clip_ratio": 0.0014231341774575412, | |
| "completion_length": 797.8333740234375, | |
| "epoch": 0.008805559442190197, | |
| "grad_norm": 0.09828333556652069, | |
| "kl": 0.0361328125, | |
| "learning_rate": 5.369057478631359e-07, | |
| "loss": 0.0535, | |
| "num_tokens": 5343864.0, | |
| "reward": 0.22396931797266006, | |
| "reward_std": 0.12565985321998596, | |
| "rewards/code_reward": 0.12938596308231354, | |
| "rewards/format_reward": 0.945833295583725, | |
| "step": 472 | |
| }, | |
| { | |
| "clip_ratio": 0.0012132310948800296, | |
| "epoch": 0.008824215288466023, | |
| "grad_norm": 0.10423380881547928, | |
| "kl": 0.0355224609375, | |
| "learning_rate": 5.343233001836694e-07, | |
| "loss": 0.0532, | |
| "step": 473 | |
| }, | |
| { | |
| "clip_ratio": 0.001151920121628791, | |
| "epoch": 0.00884287113474185, | |
| "grad_norm": 0.10031603276729584, | |
| "kl": 0.03497314453125, | |
| "learning_rate": 5.318338178609754e-07, | |
| "loss": 0.0533, | |
| "step": 474 | |
| }, | |
| { | |
| "clip_ratio": 0.001284417463466525, | |
| "completion_length": 560.7083435058594, | |
| "epoch": 0.008861526981017677, | |
| "grad_norm": 0.10862758755683899, | |
| "kl": 0.05657958984375, | |
| "learning_rate": 5.294374053487459e-07, | |
| "loss": -0.0221, | |
| "num_tokens": 5366837.0, | |
| "reward": 0.7270297110080719, | |
| "reward_std": 0.4961380064487457, | |
| "rewards/code_reward": 0.6303630471229553, | |
| "rewards/format_reward": 0.9666666388511658, | |
| "step": 475 | |
| }, | |
| { | |
| "clip_ratio": 0.0007715776737313718, | |
| "epoch": 0.008880182827293503, | |
| "grad_norm": 0.10750824958086014, | |
| "kl": 0.05804443359375, | |
| "learning_rate": 5.271341631956511e-07, | |
| "loss": -0.0221, | |
| "step": 476 | |
| }, | |
| { | |
| "clip_ratio": 0.0011225882335565984, | |
| "epoch": 0.00889883867356933, | |
| "grad_norm": 0.10623571276664734, | |
| "kl": 0.05712890625, | |
| "learning_rate": 5.249241880411181e-07, | |
| "loss": -0.0221, | |
| "step": 477 | |
| }, | |
| { | |
| "clip_ratio": 0.0007046593527775258, | |
| "completion_length": 761.6250305175781, | |
| "epoch": 0.008917494519845156, | |
| "grad_norm": 0.10407714545726776, | |
| "kl": 0.0296630859375, | |
| "learning_rate": 5.228075726112785e-07, | |
| "loss": 0.0165, | |
| "num_tokens": 5401616.0, | |
| "reward": 0.28155867755413055, | |
| "reward_std": 0.2833222597837448, | |
| "rewards/code_reward": 0.18364198692142963, | |
| "rewards/format_reward": 0.9791666567325592, | |
| "step": 478 | |
| }, | |
| { | |
| "clip_ratio": 0.0008142671431414783, | |
| "epoch": 0.008936150366120983, | |
| "grad_norm": 0.10599526017904282, | |
| "kl": 0.02685546875, | |
| "learning_rate": 5.207844057150768e-07, | |
| "loss": 0.0164, | |
| "step": 479 | |
| }, | |
| { | |
| "clip_ratio": 0.0007827770750736818, | |
| "epoch": 0.00895480621239681, | |
| "grad_norm": 0.10547417402267456, | |
| "kl": 0.02947998046875, | |
| "learning_rate": 5.188547722405437e-07, | |
| "loss": 0.0163, | |
| "step": 480 | |
| }, | |
| { | |
| "clip_ratio": 0.0009513153345324099, | |
| "completion_length": 1125.7500305175781, | |
| "epoch": 0.008973462058672636, | |
| "grad_norm": 0.10954833775758743, | |
| "kl": 0.014617919921875, | |
| "learning_rate": 5.170187531512351e-07, | |
| "loss": 0.0535, | |
| "num_tokens": 5448530.0, | |
| "reward": 0.08833334222435951, | |
| "reward_std": 0.054226743057370186, | |
| "rewards/code_reward": 0.0, | |
| "rewards/format_reward": 0.8833333253860474, | |
| "step": 481 | |
| }, | |
| { | |
| "clip_ratio": 0.0012509520165622234, | |
| "epoch": 0.008992117904948463, | |
| "grad_norm": 0.09105175733566284, | |
| "kl": 0.01336669921875, | |
| "learning_rate": 5.152764254828348e-07, | |
| "loss": 0.0535, | |
| "step": 482 | |
| }, | |
| { | |
| "clip_ratio": 0.0008532060019206256, | |
| "epoch": 0.009010773751224289, | |
| "grad_norm": 0.09203584492206573, | |
| "kl": 0.01513671875, | |
| "learning_rate": 5.136278623399225e-07, | |
| "loss": 0.0533, | |
| "step": 483 | |
| }, | |
| { | |
| "clip_ratio": 0.0007692791696172208, | |
| "completion_length": 997.0833740234375, | |
| "epoch": 0.009029429597500117, | |
| "grad_norm": 0.09475655108690262, | |
| "kl": 0.03662109375, | |
| "learning_rate": 5.120731328929058e-07, | |
| "loss": -0.0121, | |
| "num_tokens": 5492116.0, | |
| "reward": 0.5058333426713943, | |
| "reward_std": 0.3466748893260956, | |
| "rewards/code_reward": 0.40416665747761726, | |
| "rewards/format_reward": 1.0166666209697723, | |
| "step": 484 | |
| }, | |
| { | |
| "clip_ratio": 0.000924331892747432, | |
| "epoch": 0.009048085443775944, | |
| "grad_norm": 0.09625120460987091, | |
| "kl": 0.033935546875, | |
| "learning_rate": 5.106123023751187e-07, | |
| "loss": -0.0121, | |
| "step": 485 | |
| }, | |
| { | |
| "clip_ratio": 0.0005081451236037537, | |
| "epoch": 0.00906674129005177, | |
| "grad_norm": 0.08786183595657349, | |
| "kl": 0.03375244140625, | |
| "learning_rate": 5.092454320800833e-07, | |
| "loss": -0.0123, | |
| "step": 486 | |
| }, | |
| { | |
| "clip_ratio": 0.00082587351789698, | |
| "completion_length": 691.6250305175781, | |
| "epoch": 0.009085397136327597, | |
| "grad_norm": 0.08786230534315109, | |
| "kl": 0.0277099609375, | |
| "learning_rate": 5.079725793589405e-07, | |
| "loss": 0.0248, | |
| "num_tokens": 5518999.0, | |
| "reward": 0.9639226198196411, | |
| "reward_std": 0.33985109627246857, | |
| "rewards/code_reward": 0.861005961894989, | |
| "rewards/format_reward": 1.0291666388511658, | |
| "step": 487 | |
| }, | |
| { | |
| "clip_ratio": 0.0007328885840252042, | |
| "epoch": 0.009104052982603424, | |
| "grad_norm": 0.08928890526294708, | |
| "kl": 0.02886962890625, | |
| "learning_rate": 5.067937976180407e-07, | |
| "loss": 0.0248, | |
| "step": 488 | |
| }, | |
| { | |
| "clip_ratio": 0.0006209511193446815, | |
| "epoch": 0.00912270882887925, | |
| "grad_norm": 0.08875181525945663, | |
| "kl": 0.02874755859375, | |
| "learning_rate": 5.057091363167046e-07, | |
| "loss": 0.0247, | |
| "step": 489 | |
| }, | |
| { | |
| "clip_ratio": 0.0006376176825142466, | |
| "completion_length": 884.0000610351562, | |
| "epoch": 0.009141364675155077, | |
| "grad_norm": 0.0868891254067421, | |
| "kl": 0.0316162109375, | |
| "learning_rate": 5.047186409651489e-07, | |
| "loss": 0.0218, | |
| "num_tokens": 5557063.0, | |
| "reward": 0.3246118724346161, | |
| "reward_std": 0.20474953204393387, | |
| "rewards/code_reward": 0.25627854466438293, | |
| "rewards/format_reward": 0.6833333075046539, | |
| "step": 490 | |
| }, | |
| { | |
| "clip_ratio": 0.0008357224578503519, | |
| "epoch": 0.009160020521430903, | |
| "grad_norm": 0.08658529072999954, | |
| "kl": 0.03106689453125, | |
| "learning_rate": 5.038223531225742e-07, | |
| "loss": 0.022, | |
| "step": 491 | |
| }, | |
| { | |
| "clip_ratio": 0.0008412750175921246, | |
| "epoch": 0.00917867636770673, | |
| "grad_norm": 0.08690661936998367, | |
| "kl": 0.0299072265625, | |
| "learning_rate": 5.030203103954232e-07, | |
| "loss": 0.022, | |
| "step": 492 | |
| }, | |
| { | |
| "clip_ratio": 0.000971353001659736, | |
| "completion_length": 961.7916870117188, | |
| "epoch": 0.009197332213982556, | |
| "grad_norm": 0.08556245267391205, | |
| "kl": 0.019775390625, | |
| "learning_rate": 5.023125464358026e-07, | |
| "loss": -0.0011, | |
| "num_tokens": 5594090.0, | |
| "reward": 0.2715613692998886, | |
| "reward_std": 0.2223658785223961, | |
| "rewards/code_reward": 0.18156136944890022, | |
| "rewards/format_reward": 0.9000000357627869, | |
| "step": 493 | |
| }, | |
| { | |
| "clip_ratio": 0.0009858837001957, | |
| "epoch": 0.009215988060258383, | |
| "grad_norm": 0.08375484496355057, | |
| "kl": 0.01922607421875, | |
| "learning_rate": 5.016990909400709e-07, | |
| "loss": -0.0013, | |
| "step": 494 | |
| }, | |
| { | |
| "clip_ratio": 0.000899194972589612, | |
| "epoch": 0.00923464390653421, | |
| "grad_norm": 0.08476914465427399, | |
| "kl": 0.02081298828125, | |
| "learning_rate": 5.011799696475915e-07, | |
| "loss": -0.0011, | |
| "step": 495 | |
| }, | |
| { | |
| "clip_ratio": 0.0013499458436854184, | |
| "completion_length": 723.0000305175781, | |
| "epoch": 0.009253299752810036, | |
| "grad_norm": 0.11245770007371902, | |
| "kl": 0.0792236328125, | |
| "learning_rate": 5.007552043396547e-07, | |
| "loss": 0.0186, | |
| "num_tokens": 5628218.0, | |
| "reward": 0.19834937155246735, | |
| "reward_std": 0.12879429571330547, | |
| "rewards/code_reward": 0.11418268829584122, | |
| "rewards/format_reward": 0.8416666686534882, | |
| "step": 496 | |
| }, | |
| { | |
| "clip_ratio": 0.0017962601850740612, | |
| "epoch": 0.009271955599085864, | |
| "grad_norm": 0.1085243970155716, | |
| "kl": 0.07861328125, | |
| "learning_rate": 5.004248128385618e-07, | |
| "loss": 0.019, | |
| "step": 497 | |
| }, | |
| { | |
| "clip_ratio": 0.001379593217279762, | |
| "epoch": 0.009290611445361691, | |
| "grad_norm": 0.110112763941288, | |
| "kl": 0.0782470703125, | |
| "learning_rate": 5.001888090068784e-07, | |
| "loss": 0.0188, | |
| "step": 498 | |
| }, | |
| { | |
| "clip_ratio": 0.0009661116928327829, | |
| "completion_length": 820.2083435058594, | |
| "epoch": 0.009309267291637517, | |
| "grad_norm": 0.09085725247859955, | |
| "kl": 0.030242919921875, | |
| "learning_rate": 5.000472027468528e-07, | |
| "loss": 0.0247, | |
| "num_tokens": 5666887.0, | |
| "reward": 0.0845833346247673, | |
| "reward_std": 0.05373080633580685, | |
| "rewards/code_reward": 0.0, | |
| "rewards/format_reward": 0.8458333313465118, | |
| "step": 499 | |
| }, | |
| { | |
| "clip_ratio": 0.0010680440173018724, | |
| "epoch": 0.009327923137913344, | |
| "grad_norm": 0.08862722665071487, | |
| "kl": 0.02752685546875, | |
| "learning_rate": 5.000000000000001e-07, | |
| "loss": 0.0244, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.009327923137913344, | |
| "step": 500, | |
| "total_flos": 0.0, | |
| "train_loss": 0.010072903922897239, | |
| "train_runtime": 28596.067, | |
| "train_samples_per_second": 0.42, | |
| "train_steps_per_second": 0.017 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 500, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |