diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,5545 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.009327923137913344,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 982.3750610351562,
+      "epoch": 1.865584627582669e-05,
+      "grad_norm": 0.06827232986688614,
+      "kl": 0.0,
+      "learning_rate": 3.3333333333333335e-07,
+      "loss": 0.0163,
+      "num_tokens": 39753.0,
+      "reward": 0.2668055621907115,
+      "reward_std": 0.258365576621145,
+      "rewards/code_reward": 0.2638888955116272,
+      "rewards/format_reward": 0.02916666679084301,
+      "step": 1
+    },
+    {
+      "clip_ratio": 0.0,
+      "epoch": 3.731169255165338e-05,
+      "grad_norm": 0.06825702637434006,
+      "kl": 0.0,
+      "learning_rate": 6.666666666666667e-07,
+      "loss": 0.0163,
+      "step": 2
+    },
+    {
+      "clip_ratio": 0.0010449413384776562,
+      "epoch": 5.596753882748006e-05,
+      "grad_norm": 0.06731275469064713,
+      "kl": 0.00015044212341308594,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.0165,
+      "step": 3
+    },
+    {
+      "clip_ratio": 0.0009167591924779117,
+      "completion_length": 1241.1250610351562,
+      "epoch": 7.462338510330675e-05,
+      "grad_norm": 0.06577128916978836,
+      "kl": 0.00014638900756835938,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": -0.0213,
+      "num_tokens": 87072.0,
+      "reward": 0.1686033308506012,
+      "reward_std": 0.2493794783949852,
+      "rewards/code_reward": 0.1686033308506012,
+      "rewards/format_reward": 0.0,
+      "step": 4
+    },
+    {
+      "clip_ratio": 0.0010596313513815403,
+      "epoch": 9.327923137913344e-05,
+      "grad_norm": 0.06366928666830063,
+      "kl": 0.00015687942504882812,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": -0.0213,
+      "step": 5
+    },
+    {
+      "clip_ratio": 0.00101625599199906,
+      "epoch": 0.00011193507765496012,
+      "grad_norm": 0.06929203122854233,
+      "kl": 0.00015020370483398438,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": -0.0213,
+      "step": 6
+    },
+    {
+      "clip_ratio": 0.0005019755335524678,
+      "completion_length": 607.7500305175781,
+      "epoch": 0.0001305909239307868,
+      "grad_norm": 0.054728299379348755,
+      "kl": 0.00021600723266601562,
+      "learning_rate": 2.3333333333333336e-06,
+      "loss": 0.0112,
+      "num_tokens": 110922.0,
+      "reward": 0.4583333432674408,
+      "reward_std": 0.14433756470680237,
+      "rewards/code_reward": 0.4583333432674408,
+      "rewards/format_reward": 0.0,
+      "step": 7
+    },
+    {
+      "clip_ratio": 0.0005628917133435607,
+      "epoch": 0.0001492467702066135,
+      "grad_norm": 0.05518824979662895,
+      "kl": 0.00023031234741210938,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.011,
+      "step": 8
+    },
+    {
+      "clip_ratio": 0.0006851793150417507,
+      "epoch": 0.00016790261648244018,
+      "grad_norm": 0.05371546372771263,
+      "kl": 0.00021839141845703125,
+      "learning_rate": 3e-06,
+      "loss": 0.0113,
+      "step": 9
+    },
+    {
+      "clip_ratio": 0.0009316161158494651,
+      "completion_length": 763.25,
+      "epoch": 0.00018655846275826687,
+      "grad_norm": 0.06738901138305664,
+      "kl": 0.00017595291137695312,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": -0.017,
+      "num_tokens": 140040.0,
+      "reward": 0.29250000417232513,
+      "reward_std": 0.4735611826181412,
+      "rewards/code_reward": 0.2916666716337204,
+      "rewards/format_reward": 0.008333333767950535,
+      "step": 10
+    },
+    {
+      "clip_ratio": 0.0005913192144362256,
+      "epoch": 0.00020521430903409357,
+      "grad_norm": 0.07004611194133759,
+      "kl": 0.0001888275146484375,
+      "learning_rate": 3.6666666666666666e-06,
+      "loss": -0.0171,
+      "step": 11
+    },
+    {
+      "clip_ratio": 0.000849212781758979,
+      "epoch": 0.00022387015530992023,
+      "grad_norm": 0.069913350045681,
+      "kl": 0.00017833709716796875,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": -0.0172,
+      "step": 12
+    },
+    {
+      "clip_ratio": 0.0005434846971184015,
+      "completion_length": 622.1250305175781,
+      "epoch": 0.00024252600158574693,
+      "grad_norm": 0.061467599123716354,
+      "kl": 0.00020265579223632812,
+      "learning_rate": 4.333333333333334e-06,
+      "loss": 0.0123,
+      "num_tokens": 165519.0,
+      "reward": 0.31522059440612793,
+      "reward_std": 0.22002992033958435,
+      "rewards/code_reward": 0.3139705955982208,
+      "rewards/format_reward": 0.012500000186264515,
+      "step": 13
+    },
+    {
+      "clip_ratio": 0.0004921667859889567,
+      "epoch": 0.0002611818478615736,
+      "grad_norm": 0.0641019269824028,
+      "kl": 0.000213623046875,
+      "learning_rate": 4.666666666666667e-06,
+      "loss": 0.0124,
+      "step": 14
+    },
+    {
+      "clip_ratio": 0.0007367254002019763,
+      "epoch": 0.0002798376941374003,
+      "grad_norm": 0.13394594192504883,
+      "kl": 0.00019359588623046875,
+      "learning_rate": 5e-06,
+      "loss": 0.0122,
+      "step": 15
+    },
+    {
+      "clip_ratio": 0.0015055734547786415,
+      "completion_length": 809.2500305175781,
+      "epoch": 0.000298493540413227,
+      "grad_norm": 0.08201699703931808,
+      "kl": 0.00023174285888671875,
+      "learning_rate": 4.999952797253148e-06,
+      "loss": 0.009,
+      "num_tokens": 200625.0,
+      "reward": 0.363928597420454,
+      "reward_std": 0.2377874432131648,
+      "rewards/code_reward": 0.3526785671710968,
+      "rewards/format_reward": 0.11249998956918716,
+      "step": 16
+    },
+    {
+      "clip_ratio": 0.0011205525370314717,
+      "epoch": 0.0003171493866890537,
+      "grad_norm": 0.08107728511095047,
+      "kl": 0.00022745132446289062,
+      "learning_rate": 4.9998111909931225e-06,
+      "loss": 0.0088,
+      "step": 17
+    },
+    {
+      "clip_ratio": 0.0010780769807752222,
+      "epoch": 0.00033580523296488035,
+      "grad_norm": 0.08552956581115723,
+      "kl": 0.00022220611572265625,
+      "learning_rate": 4.999575187161439e-06,
+      "loss": 0.0088,
+      "step": 18
+    },
+    {
+      "clip_ratio": 0.0009002784790936857,
+      "completion_length": 777.0416870117188,
+      "epoch": 0.0003544610792407071,
+      "grad_norm": 0.08441054821014404,
+      "kl": 0.00020885467529296875,
+      "learning_rate": 4.9992447956603455e-06,
+      "loss": -0.0108,
+      "num_tokens": 234130.0,
+      "reward": 0.6849345862865448,
+      "reward_std": 0.30150339007377625,
+      "rewards/code_reward": 0.6828512400388718,
+      "rewards/format_reward": 0.02083333395421505,
+      "step": 19
+    },
+    {
+      "clip_ratio": 0.0011175140971317887,
+      "epoch": 0.00037311692551653374,
+      "grad_norm": 0.08343806117773056,
+      "kl": 0.0002079010009765625,
+      "learning_rate": 4.998820030352409e-06,
+      "loss": -0.0108,
+      "step": 20
+    },
+    {
+      "clip_ratio": 0.0009022637677844614,
+      "epoch": 0.0003917727717923604,
+      "grad_norm": 0.0833282396197319,
+      "kl": 0.00020360946655273438,
+      "learning_rate": 4.998300909059929e-06,
+      "loss": -0.0111,
+      "step": 21
+    },
+    {
+      "clip_ratio": 0.0015675597824156284,
+      "completion_length": 630.5833435058594,
+      "epoch": 0.00041042861806818713,
+      "grad_norm": 0.08936360478401184,
+      "kl": 0.00019788742065429688,
+      "learning_rate": 4.997687453564198e-06,
+      "loss": 0.0142,
+      "num_tokens": 257484.0,
+      "reward": 0.3370833722874522,
+      "reward_std": 0.252491211052984,
+      "rewards/code_reward": 0.3333333432674408,
+      "rewards/format_reward": 0.037500000558793545,
+      "step": 22
+    },
+    {
+      "clip_ratio": 0.001083480688976124,
+      "epoch": 0.0004290844643440138,
+      "grad_norm": 0.08784651011228561,
+      "kl": 0.000202178955078125,
+      "learning_rate": 4.9969796896045775e-06,
+      "loss": 0.014,
+      "step": 23
+    },
+    {
+      "clip_ratio": 0.0011372591252438724,
+      "epoch": 0.00044774031061984047,
+      "grad_norm": 0.08748355507850647,
+      "kl": 0.000209808349609375,
+      "learning_rate": 4.996177646877426e-06,
+      "loss": 0.014,
+      "step": 24
+    },
+    {
+      "clip_ratio": 0.0014254440320655704,
+      "completion_length": 834.6250305175781,
+      "epoch": 0.0004663961568956672,
+      "grad_norm": 0.07256177812814713,
+      "kl": 0.00018978118896484375,
+      "learning_rate": 4.995281359034851e-06,
+      "loss": 0.0032,
+      "num_tokens": 297543.0,
+      "reward": 0.20448413118720055,
+      "reward_std": 0.30803024023771286,
+      "rewards/code_reward": 0.20198413357138634,
+      "rewards/format_reward": 0.025000001303851604,
+      "step": 25
+    },
+    {
+      "clip_ratio": 0.0013226708979345858,
+      "epoch": 0.00048505200317149386,
+      "grad_norm": 0.07238035649061203,
+      "kl": 0.00017786026000976562,
+      "learning_rate": 4.994290863683296e-06,
+      "loss": 0.0033,
+      "step": 26
+    },
+    {
+      "clip_ratio": 0.0009347930317744613,
+      "epoch": 0.0005037078494473206,
+      "grad_norm": 0.07114218175411224,
+      "kl": 0.00019788742065429688,
+      "learning_rate": 4.99320620238196e-06,
+      "loss": 0.0033,
+      "step": 27
+    },
+    {
+      "clip_ratio": 0.0008377690683118999,
+      "completion_length": 899.4166870117188,
+      "epoch": 0.0005223636957231473,
+      "grad_norm": 0.12414078414440155,
+      "kl": 0.00019073486328125,
+      "learning_rate": 4.99202742064106e-06,
+      "loss": 0.0106,
+      "num_tokens": 331453.0,
+      "reward": 0.15678573166951537,
+      "reward_std": 0.17976204119622707,
+      "rewards/code_reward": 0.1517857164144516,
+      "rewards/format_reward": 0.05000000074505806,
+      "step": 28
+    },
+    {
+      "clip_ratio": 0.0005996464460622519,
+      "epoch": 0.0005410195419989739,
+      "grad_norm": 0.07990814745426178,
+      "kl": 0.00018262863159179688,
+      "learning_rate": 4.990754567919917e-06,
+      "loss": 0.0103,
+      "step": 29
+    },
+    {
+      "clip_ratio": 0.0008573587401770055,
+      "epoch": 0.0005596753882748006,
+      "grad_norm": 0.0788721814751625,
+      "kl": 0.0001926422119140625,
+      "learning_rate": 4.989387697624881e-06,
+      "loss": 0.0104,
+      "step": 30
+    },
+    {
+      "clip_ratio": 0.0007707980694249272,
+      "completion_length": 873.875,
+      "epoch": 0.0005783312345506273,
+      "grad_norm": 0.07595472037792206,
+      "kl": 0.0002765655517578125,
+      "learning_rate": 4.987926867107095e-06,
+      "loss": 0.0066,
+      "num_tokens": 368998.0,
+      "reward": 0.4440600574016571,
+      "reward_std": 0.30831071734428406,
+      "rewards/code_reward": 0.43614333122968674,
+      "rewards/format_reward": 0.07916666753590107,
+      "step": 31
+    },
+    {
+      "clip_ratio": 0.0011201543966308236,
+      "epoch": 0.000596987080826454,
+      "grad_norm": 0.0760602131485939,
+      "kl": 0.0002522468566894531,
+      "learning_rate": 4.986372137660078e-06,
+      "loss": 0.0069,
+      "step": 32
+    },
+    {
+      "clip_ratio": 0.0011669989908114076,
+      "epoch": 0.0006156429271022807,
+      "grad_norm": 0.07646892964839935,
+      "kl": 0.00030231475830078125,
+      "learning_rate": 4.984723574517165e-06,
+      "loss": 0.0063,
+      "step": 33
+    },
+    {
+      "clip_ratio": 0.0003657292982097715,
+      "completion_length": 751.3333435058594,
+      "epoch": 0.0006342987733781074,
+      "grad_norm": 0.05740216001868248,
+      "kl": 0.00026035308837890625,
+      "learning_rate": 4.9829812468487655e-06,
+      "loss": -0.0053,
+      "num_tokens": 402006.0,
+      "reward": 0.7125000059604645,
+      "reward_std": 0.20762184262275696,
+      "rewards/code_reward": 0.7083333283662796,
+      "rewards/format_reward": 0.0416666679084301,
+      "step": 34
+    },
+    {
+      "clip_ratio": 0.00030940253054723144,
+      "epoch": 0.000652954619653934,
+      "grad_norm": 0.0546778179705143,
+      "kl": 0.00029468536376953125,
+      "learning_rate": 4.981145227759457e-06,
+      "loss": -0.0054,
+      "step": 35
+    },
+    {
+      "clip_ratio": 0.00020408157433848828,
+      "epoch": 0.0006716104659297607,
+      "grad_norm": 0.05456351861357689,
+      "kl": 0.0002956390380859375,
+      "learning_rate": 4.979215594284924e-06,
+      "loss": -0.0054,
+      "step": 36
+    },
+    {
+      "clip_ratio": 0.0005608744686469436,
+      "completion_length": 875.6667175292969,
+      "epoch": 0.0006902663122055874,
+      "grad_norm": 0.06957173347473145,
+      "kl": 0.00028324127197265625,
+      "learning_rate": 4.977192427388722e-06,
+      "loss": 0.0157,
+      "num_tokens": 444430.0,
+      "reward": 0.515416668727994,
+      "reward_std": 0.0217901524156332,
+      "rewards/code_reward": 0.5,
+      "rewards/format_reward": 0.15416665375232697,
+      "step": 37
+    },
+    {
+      "clip_ratio": 0.0005198217695578933,
+      "epoch": 0.0007089221584814141,
+      "grad_norm": 0.06720055639743805,
+      "kl": 0.0002918243408203125,
+      "learning_rate": 4.9750758119588824e-06,
+      "loss": 0.0157,
+      "step": 38
+    },
+    {
+      "clip_ratio": 0.0003698411746881902,
+      "epoch": 0.0007275780047572408,
+      "grad_norm": 0.06269616633653641,
+      "kl": 0.00034332275390625,
+      "learning_rate": 4.972865836804349e-06,
+      "loss": 0.0156,
+      "step": 39
+    },
+    {
+      "clip_ratio": 0.0008287752571050078,
+      "completion_length": 668.0416870117188,
+      "epoch": 0.0007462338510330675,
+      "grad_norm": 0.10391846299171448,
+      "kl": 0.00041675567626953125,
+      "learning_rate": 4.970562594651254e-06,
+      "loss": 0.066,
+      "num_tokens": 469319.0,
+      "reward": 0.3462119158357382,
+      "reward_std": 0.24788545817136765,
+      "rewards/code_reward": 0.32662858441472054,
+      "rewards/format_reward": 0.1958333309739828,
+      "step": 40
+    },
+    {
+      "clip_ratio": 0.0010252603678964078,
+      "epoch": 0.0007648896973088942,
+      "grad_norm": 0.10473919659852982,
+      "kl": 0.00048065185546875,
+      "learning_rate": 4.968166182139026e-06,
+      "loss": 0.0657,
+      "step": 41
+    },
+    {
+      "clip_ratio": 0.0008386686386074871,
+      "epoch": 0.0007835455435847208,
+      "grad_norm": 0.10585241764783859,
+      "kl": 0.0005102157592773438,
+      "learning_rate": 4.9656766998163306e-06,
+      "loss": 0.0657,
+      "step": 42
+    },
+    {
+      "clip_ratio": 0.0009505777561571449,
+      "completion_length": 773.8750305175781,
+      "epoch": 0.0008022013898605476,
+      "grad_norm": 0.08152071386575699,
+      "kl": 0.0006093978881835938,
+      "learning_rate": 4.963094252136865e-06,
+      "loss": 0.0317,
+      "num_tokens": 500552.0,
+      "reward": 0.46135416626930237,
+      "reward_std": 0.2722689062356949,
+      "rewards/code_reward": 0.44843751192092896,
+      "rewards/format_reward": 0.12916667759418488,
+      "step": 43
+    },
+    {
+      "clip_ratio": 0.001228984270710498,
+      "epoch": 0.0008208572361363743,
+      "grad_norm": 0.08072768896818161,
+      "kl": 0.0006775856018066406,
+      "learning_rate": 4.960418947454958e-06,
+      "loss": 0.0318,
+      "step": 44
+    },
+    {
+      "clip_ratio": 0.001452101394534111,
+      "epoch": 0.0008395130824122009,
+      "grad_norm": 0.07877793163061142,
+      "kl": 0.0007939338684082031,
+      "learning_rate": 4.957650898021038e-06,
+      "loss": 0.0316,
+      "step": 45
+    },
+    {
+      "clip_ratio": 0.0013901306956540793,
+      "completion_length": 523.6666870117188,
+      "epoch": 0.0008581689286880276,
+      "grad_norm": 0.09457489103078842,
+      "kl": 0.0012969970703125,
+      "learning_rate": 4.954790219976915e-06,
+      "loss": -0.0098,
+      "num_tokens": 523440.0,
+      "reward": 0.4345560818910599,
+      "reward_std": 0.32685674726963043,
+      "rewards/code_reward": 0.4170560836791992,
+      "rewards/format_reward": 0.17500000074505806,
+      "step": 46
+    },
+    {
+      "clip_ratio": 0.0014728345850016922,
+      "epoch": 0.0008768247749638543,
+      "grad_norm": 0.09064733982086182,
+      "kl": 0.00167083740234375,
+      "learning_rate": 4.95183703335091e-06,
+      "loss": -0.0102,
+      "step": 47
+    },
+    {
+      "clip_ratio": 0.0014083714631851763,
+      "epoch": 0.0008954806212396809,
+      "grad_norm": 0.09516453742980957,
+      "kl": 0.002044677734375,
+      "learning_rate": 4.948791462052819e-06,
+      "loss": -0.0104,
+      "step": 48
+    },
+    {
+      "clip_ratio": 0.0019310088246129453,
+      "completion_length": 755.7083740234375,
+      "epoch": 0.0009141364675155077,
+      "grad_norm": 0.07378407567739487,
+      "kl": 0.0006542205810546875,
+      "learning_rate": 4.945653633868716e-06,
+      "loss": 0.0031,
+      "num_tokens": 552221.0,
+      "reward": 0.5968180000782013,
+      "reward_std": 0.36423343420028687,
+      "rewards/code_reward": 0.5947346538305283,
+      "rewards/format_reward": 0.02083333395421505,
+      "step": 49
+    },
+    {
+      "clip_ratio": 0.0012188139371573925,
+      "epoch": 0.0009327923137913344,
+      "grad_norm": 0.07792558521032333,
+      "kl": 0.0006256103515625,
+      "learning_rate": 4.942423680455584e-06,
+      "loss": 0.0033,
+      "step": 50
+    },
+    {
+      "clip_ratio": 0.0013152067258488387,
+      "epoch": 0.000951448160067161,
+      "grad_norm": 0.07279552519321442,
+      "kl": 0.000690460205078125,
+      "learning_rate": 4.939101737335802e-06,
+      "loss": 0.003,
+      "step": 51
+    },
+    {
+      "clip_ratio": 0.0012177950120531023,
+      "completion_length": 1433.25,
+      "epoch": 0.0009701040063429877,
+      "grad_norm": 0.0953638032078743,
+      "kl": 0.0007143020629882812,
+      "learning_rate": 4.935687943891447e-06,
+      "loss": -0.0997,
+      "num_tokens": 610451.0,
+      "reward": 0.01875000074505806,
+      "reward_std": 0.020104273222386837,
+      "rewards/code_reward": 0.0,
+      "rewards/format_reward": 0.1874999888241291,
+      "step": 52
+    },
+    {
+      "clip_ratio": 0.0008826107368804514,
+      "epoch": 0.0009887598526188144,
+      "grad_norm": 0.07305457442998886,
+      "kl": 0.0007915496826171875,
+      "learning_rate": 4.932182443358458e-06,
+      "loss": -0.0999,
+      "step": 53
+    },
+    {
+      "clip_ratio": 0.0009481910965405405,
+      "epoch": 0.0010074156988946412,
+      "grad_norm": 0.1027429923415184,
+      "kl": 0.00098419189453125,
+      "learning_rate": 4.928585382820616e-06,
+      "loss": -0.0996,
+      "step": 54
+    },
+    {
+      "clip_ratio": 0.0013086027465760708,
+      "completion_length": 899.5416870117188,
+      "epoch": 0.0010260715451704677,
+      "grad_norm": 0.07541598379611969,
+      "kl": 0.00274658203125,
+      "learning_rate": 4.924896913203376e-06,
+      "loss": -0.0237,
+      "num_tokens": 648072.0,
+      "reward": 0.2216584151610732,
+      "reward_std": 0.22969772201031446,
+      "rewards/code_reward": 0.20915842056274414,
+      "rewards/format_reward": 0.125,
+      "step": 55
+    },
+    {
+      "clip_ratio": 0.0009090554085560143,
+      "epoch": 0.0010447273914462945,
+      "grad_norm": 0.07311614602804184,
+      "kl": 0.003662109375,
+      "learning_rate": 4.921117189267535e-06,
+      "loss": -0.0236,
+      "step": 56
+    },
+    {
+      "clip_ratio": 0.0012668125564232469,
+      "epoch": 0.0010633832377221213,
+      "grad_norm": 0.07483425736427307,
+      "kl": 0.00545501708984375,
+      "learning_rate": 4.917246369602742e-06,
+      "loss": -0.0237,
+      "step": 57
+    },
+    {
+      "clip_ratio": 0.0015675598988309503,
+      "completion_length": 691.2916870117188,
+      "epoch": 0.0010820390839979478,
+      "grad_norm": 0.09265889972448349,
+      "kl": 0.0068817138671875,
+      "learning_rate": 4.9132846166208355e-06,
+      "loss": -0.0068,
+      "num_tokens": 678547.0,
+      "reward": 0.2671003444120288,
+      "reward_std": 0.2649538954719901,
+      "rewards/code_reward": 0.2508503496646881,
+      "rewards/format_reward": 0.16249999776482582,
+      "step": 58
+    },
+    {
+      "clip_ratio": 0.0011916961520910263,
+      "epoch": 0.0011006949302737746,
+      "grad_norm": 0.07879021763801575,
+      "kl": 0.010772705078125,
+      "learning_rate": 4.9092320965490365e-06,
+      "loss": -0.0072,
+      "step": 59
+    },
+    {
+      "clip_ratio": 0.0013531986624002457,
+      "epoch": 0.0011193507765496012,
+      "grad_norm": 0.08185919374227524,
+      "kl": 0.013824462890625,
+      "learning_rate": 4.905088979422971e-06,
+      "loss": -0.0072,
+      "step": 60
+    },
+    {
+      "clip_ratio": 0.001054943015333265,
+      "completion_length": 816.0833740234375,
+      "epoch": 0.001138006622825428,
+      "grad_norm": 0.09033277630805969,
+      "kl": 0.011138916015625,
+      "learning_rate": 4.900855439079536e-06,
+      "loss": 0.0434,
+      "num_tokens": 708969.0,
+      "reward": 0.13236112147569656,
+      "reward_std": 0.17511780560016632,
+      "rewards/code_reward": 0.1111111119389534,
+      "rewards/format_reward": 0.21250000223517418,
+      "step": 61
+    },
+    {
+      "clip_ratio": 0.001191994990222156,
+      "epoch": 0.0011566624691012545,
+      "grad_norm": 0.11616651713848114,
+      "kl": 0.0153656005859375,
+      "learning_rate": 4.8965316531496055e-06,
+      "loss": 0.0433,
+      "step": 62
+    },
+    {
+      "clip_ratio": 0.0013809394440613687,
+      "epoch": 0.0011753183153770813,
+      "grad_norm": 0.08028258383274078,
+      "kl": 0.0198516845703125,
+      "learning_rate": 4.892117803050578e-06,
+      "loss": 0.0432,
+      "step": 63
+    },
+    {
+      "clip_ratio": 0.0014273470733314753,
+      "completion_length": 478.12501525878906,
+      "epoch": 0.001193974161652908,
+      "grad_norm": 0.12240626662969589,
+      "kl": 0.02911376953125,
+      "learning_rate": 4.887614073978761e-06,
+      "loss": 0.0251,
+      "num_tokens": 727836.0,
+      "reward": 0.5829170048236847,
+      "reward_std": 0.42754843831062317,
+      "rewards/code_reward": 0.5400003343820572,
+      "rewards/format_reward": 0.42916665971279144,
+      "step": 64
+    },
+    {
+      "clip_ratio": 0.001149984309449792,
+      "epoch": 0.0012126300079287346,
+      "grad_norm": 0.11828020960092545,
+      "kl": 0.0277099609375,
+      "learning_rate": 4.883020654901609e-06,
+      "loss": 0.0248,
+      "step": 65
+    },
+    {
+      "clip_ratio": 0.001388654694892466,
+      "epoch": 0.0012312858542045614,
+      "grad_norm": 0.15795664489269257,
+      "kl": 0.02325439453125,
+      "learning_rate": 4.878337738549785e-06,
+      "loss": 0.0245,
+      "step": 66
+    },
+    {
+      "clip_ratio": 0.0012379828258417547,
+      "completion_length": 1035.7083740234375,
+      "epoch": 0.001249941700480388,
+      "grad_norm": 0.07949040085077286,
+      "kl": 0.0186614990234375,
+      "learning_rate": 4.873565521409082e-06,
+      "loss": 0.0077,
+      "num_tokens": 767177.0,
+      "reward": 0.07208333350718021,
+      "reward_std": 0.15239802561700344,
+      "rewards/code_reward": 0.0416666679084301,
+      "rewards/format_reward": 0.30416667461395264,
+      "step": 67
+    },
+    {
+      "clip_ratio": 0.0013320732396095991,
+      "epoch": 0.0012685975467562147,
+      "grad_norm": 0.0713808611035347,
+      "kl": 0.014373779296875,
+      "learning_rate": 4.868704203712173e-06,
+      "loss": 0.0076,
+      "step": 68
+    },
+    {
+      "clip_ratio": 0.0018569266539998353,
+      "epoch": 0.0012872533930320415,
+      "grad_norm": 0.08412781357765198,
+      "kl": 0.01141357421875,
+      "learning_rate": 4.86375398943021e-06,
+      "loss": 0.0076,
+      "step": 69
+    },
+    {
+      "clip_ratio": 0.0012622788199223578,
+      "completion_length": 733.0416870117188,
+      "epoch": 0.001305909239307868,
+      "grad_norm": 0.08328709751367569,
+      "kl": 0.01409912109375,
+      "learning_rate": 4.858715086264274e-06,
+      "loss": 0.0612,
+      "num_tokens": 795078.0,
+      "reward": 0.9353921413421631,
+      "reward_std": 0.17414550855755806,
+      "rewards/code_reward": 0.9053921699523926,
+      "rewards/format_reward": 0.30000001192092896,
+      "step": 70
+    },
+    {
+      "clip_ratio": 0.001602432457730174,
+      "epoch": 0.0013245650855836948,
+      "grad_norm": 0.08351202309131622,
+      "kl": 0.015625,
+      "learning_rate": 4.853587705636646e-06,
+      "loss": 0.0614,
+      "step": 71
+    },
+    {
+      "clip_ratio": 0.0013681039854418486,
+      "epoch": 0.0013432209318595214,
+      "grad_norm": 0.08263003826141357,
+      "kl": 0.01739501953125,
+      "learning_rate": 4.84837206268195e-06,
+      "loss": 0.0613,
+      "step": 72
+    },
+    {
+      "clip_ratio": 0.0006202208460308611,
+      "completion_length": 748.2083435058594,
+      "epoch": 0.0013618767781353482,
+      "grad_norm": 0.08475489169359207,
+      "kl": 0.01568603515625,
+      "learning_rate": 4.8430683762381195e-06,
+      "loss": 0.0156,
+      "num_tokens": 823439.0,
+      "reward": 0.46898147463798523,
+      "reward_std": 0.3545081913471222,
+      "rewards/code_reward": 0.4293981343507767,
+      "rewards/format_reward": 0.3958333432674408,
+      "step": 73
+    },
+    {
+      "clip_ratio": 0.000859813007991761,
+      "epoch": 0.0013805326244111747,
+      "grad_norm": 0.08259416371583939,
+      "kl": 0.016510009765625,
+      "learning_rate": 4.837676868837213e-06,
+      "loss": 0.0154,
+      "step": 74
+    },
+    {
+      "clip_ratio": 0.00038046142435632646,
+      "epoch": 0.0013991884706870015,
+      "grad_norm": 0.08513620495796204,
+      "kl": 0.01690673828125,
+      "learning_rate": 4.832197766696085e-06,
+      "loss": 0.0151,
+      "step": 75
+    },
+    {
+      "clip_ratio": 0.0009400486887898296,
+      "completion_length": 746.0833435058594,
+      "epoch": 0.0014178443169628283,
+      "grad_norm": 0.08578525483608246,
+      "kl": 0.009674072265625,
+      "learning_rate": 4.826631299706887e-06,
+      "loss": -0.0026,
+      "num_tokens": 857425.0,
+      "reward": 0.7170831263065338,
+      "reward_std": 0.29470987617969513,
+      "rewards/code_reward": 0.6824997961521149,
+      "rewards/format_reward": 0.34583334624767303,
+      "step": 76
+    },
+    {
+      "clip_ratio": 0.000981206467258744,
+      "epoch": 0.0014365001632386549,
+      "grad_norm": 0.08752106130123138,
+      "kl": 0.0097503662109375,
+      "learning_rate": 4.820977701427424e-06,
+      "loss": -0.0028,
+      "step": 77
+    },
+    {
+      "clip_ratio": 0.001091863785404712,
+      "epoch": 0.0014551560095144816,
+      "grad_norm": 0.08650875091552734,
+      "kl": 0.010467529296875,
+      "learning_rate": 4.81523720907136e-06,
+      "loss": -0.0027,
+      "step": 78
+    },
+    {
+      "clip_ratio": 0.0008625364207546227,
+      "completion_length": 843.0417175292969,
+      "epoch": 0.0014738118557903082,
+      "grad_norm": 0.08371932804584503,
+      "kl": 0.01336669921875,
+      "learning_rate": 4.809410063498254e-06,
+      "loss": -0.0214,
+      "num_tokens": 890714.0,
+      "reward": 0.28875000309199095,
+      "reward_std": 0.26341581624001265,
+      "rewards/code_reward": 0.25,
+      "rewards/format_reward": 0.38750001788139343,
+      "step": 79
+    },
+    {
+      "clip_ratio": 0.0010699788690544665,
+      "epoch": 0.001492467702066135,
+      "grad_norm": 0.08200695365667343,
+      "kl": 0.0144500732421875,
+      "learning_rate": 4.8034965092034656e-06,
+      "loss": -0.0214,
+      "step": 80
+    },
+    {
+      "clip_ratio": 0.001069951627869159,
+      "epoch": 0.0015111235483419617,
+      "grad_norm": 0.08111701160669327,
+      "kl": 0.013519287109375,
+      "learning_rate": 4.797496794307889e-06,
+      "loss": -0.0216,
+      "step": 81
+    },
+    {
+      "clip_ratio": 0.0005499664548551664,
+      "completion_length": 688.2083435058594,
+      "epoch": 0.0015297793946177883,
+      "grad_norm": 0.09052547812461853,
+      "kl": 0.0286865234375,
+      "learning_rate": 4.791411170547545e-06,
+      "loss": 0.0283,
+      "num_tokens": 923131.0,
+      "reward": 0.6477343738079071,
+      "reward_std": 0.3908188194036484,
+      "rewards/code_reward": 0.6152343600988388,
+      "rewards/format_reward": 0.32500000298023224,
+      "step": 82
+    },
+    {
+      "clip_ratio": 0.0005515173252206296,
+      "epoch": 0.001548435240893615,
+      "grad_norm": 0.08914868533611298,
+      "kl": 0.0316162109375,
+      "learning_rate": 4.785239893263017e-06,
+      "loss": 0.0282,
+      "step": 83
+    },
+    {
+      "clip_ratio": 0.0004736810806207359,
+      "epoch": 0.0015670910871694416,
+      "grad_norm": 0.08998391777276993,
+      "kl": 0.0325927734375,
+      "learning_rate": 4.778983221388742e-06,
+      "loss": 0.0282,
+      "step": 84
+    },
+    {
+      "clip_ratio": 0.0016970738070085645,
+      "completion_length": 1049.75,
+      "epoch": 0.0015857469334452684,
+      "grad_norm": 0.07974597066640854,
+      "kl": 0.0087890625,
+      "learning_rate": 4.77264141744214e-06,
+      "loss": -0.0355,
+      "num_tokens": 968365.0,
+      "reward": 0.19875000324100256,
+      "reward_std": 0.2349306456744671,
+      "rewards/code_reward": 0.1666666716337204,
+      "rewards/format_reward": 0.32083334028720856,
+      "step": 85
+    },
+    {
+      "clip_ratio": 0.001736409030854702,
+      "epoch": 0.0016044027797210952,
+      "grad_norm": 0.0792006105184555,
+      "kl": 0.009368896484375,
+      "learning_rate": 4.766214747512603e-06,
+      "loss": -0.0355,
+      "step": 86
+    },
+    {
+      "clip_ratio": 0.0015154886059463024,
+      "epoch": 0.0016230586259969218,
+      "grad_norm": 0.10796553641557693,
+      "kl": 0.009796142578125,
+      "learning_rate": 4.759703481250331e-06,
+      "loss": -0.036,
+      "step": 87
+    },
+    {
+      "clip_ratio": 0.0013903837534599006,
+      "completion_length": 792.4583435058594,
+      "epoch": 0.0016417144722727485,
+      "grad_norm": 0.19102805852890015,
+      "kl": 0.0703125,
+      "learning_rate": 4.753107891855015e-06,
+      "loss": 0.0435,
+      "num_tokens": 999360.0,
+      "reward": 0.3778717517852783,
+      "reward_std": 0.32771627604961395,
+      "rewards/code_reward": 0.3512050434947014,
+      "rewards/format_reward": 0.2666666656732559,
+      "step": 88
+    },
+    {
+      "clip_ratio": 0.0016154100885614753,
+      "epoch": 0.001660370318548575,
+      "grad_norm": 0.09603425115346909,
+      "kl": 0.0440673828125,
+      "learning_rate": 4.746428256064375e-06,
+      "loss": 0.0434,
+      "step": 89
+    },
+    {
+      "clip_ratio": 0.0018210893031209707,
+      "epoch": 0.0016790261648244019,
+      "grad_norm": 0.09434056282043457,
+      "kl": 0.0350341796875,
+      "learning_rate": 4.7396648541425534e-06,
+      "loss": 0.0433,
+      "step": 90
+    },
+    {
+      "clip_ratio": 0.0014776080497540534,
+      "completion_length": 766.6666870117188,
+      "epoch": 0.0016976820111002284,
+      "grad_norm": 0.08751969784498215,
+      "kl": 0.012847900390625,
+      "learning_rate": 4.732817969868348e-06,
+      "loss": -0.0055,
+      "num_tokens": 1032760.0,
+      "reward": 0.503094419836998,
+      "reward_std": 0.4136483669281006,
+      "rewards/code_reward": 0.4914277046918869,
+      "rewards/format_reward": 0.11666666716337204,
+      "step": 91
+    },
+    {
+      "clip_ratio": 0.0013046148815192282,
+      "epoch": 0.0017163378573760552,
+      "grad_norm": 0.08883315324783325,
+      "kl": 0.010772705078125,
+      "learning_rate": 4.7258878905233095e-06,
+      "loss": -0.0056,
+      "step": 92
+    },
+    {
+      "clip_ratio": 0.000994603120489046,
+      "epoch": 0.001734993703651882,
+      "grad_norm": 0.10364013910293579,
+      "kl": 0.010162353515625,
+      "learning_rate": 4.718874906879688e-06,
+      "loss": -0.0057,
+      "step": 93
+    },
+    {
+      "clip_ratio": 0.0011711710831150413,
+      "completion_length": 1132.4583740234375,
+      "epoch": 0.0017536495499277085,
+      "grad_norm": 0.07417519390583038,
+      "kl": 0.0053558349609375,
+      "learning_rate": 4.711779313188231e-06,
+      "loss": 0.0018,
+      "num_tokens": 1080639.0,
+      "reward": 0.04583333432674408,
+      "reward_std": 0.026794558390975,
+      "rewards/code_reward": 0.0,
+      "rewards/format_reward": 0.4583333432674408,
+      "step": 94
+    },
+    {
+      "clip_ratio": 0.00107975909486413,
+      "epoch": 0.0017723053962035353,
+      "grad_norm": 0.07358817011117935,
+      "kl": 0.005706787109375,
+      "learning_rate": 4.70460140716584e-06,
+      "loss": 0.0019,
+      "step": 95
+    },
+    {
+      "clip_ratio": 0.0008912903140299022,
+      "epoch": 0.0017909612424793619,
+      "grad_norm": 0.08615133166313171,
+      "kl": 0.0057830810546875,
+      "learning_rate": 4.697341489983076e-06,
+      "loss": 0.0019,
+      "step": 96
+    },
+    {
+      "clip_ratio": 0.002020051470026374,
+      "completion_length": 727.7500305175781,
+      "epoch": 0.0018096170887551887,
+      "grad_norm": 0.0899001881480217,
+      "kl": 0.01202392578125,
+      "learning_rate": 4.6899998662515215e-06,
+      "loss": -0.001,
+      "num_tokens": 1111077.0,
+      "reward": 0.29144802689552307,
+      "reward_std": 0.33243465423583984,
+      "rewards/code_reward": 0.2614480257034302,
+      "rewards/format_reward": 0.30000001192092896,
+      "step": 97
+    },
+    {
+      "clip_ratio": 0.0011579456622712314,
+      "epoch": 0.0018282729350310154,
+      "grad_norm": 0.09250804036855698,
+      "kl": 0.013916015625,
+      "learning_rate": 4.682576844011007e-06,
+      "loss": -0.0011,
+      "step": 98
+    },
+    {
+      "clip_ratio": 0.0017355763702653348,
+      "epoch": 0.001846928781306842,
+      "grad_norm": 0.0963953509926796,
+      "kl": 0.015106201171875,
+      "learning_rate": 4.675072734716678e-06,
+      "loss": -0.0013,
+      "step": 99
+    },
+    {
+      "clip_ratio": 0.0009290422021877021,
+      "completion_length": 1036.0833740234375,
+      "epoch": 0.0018655846275826688,
+      "grad_norm": 0.09167575091123581,
+      "kl": 0.015167236328125,
+      "learning_rate": 4.667487853225931e-06,
+      "loss": -0.0103,
+      "num_tokens": 1155731.0,
+      "reward": 0.11254538409411907,
+      "reward_std": 0.16347683407366276,
+      "rewards/code_reward": 0.07879538089036942,
+      "rewards/format_reward": 0.3374999910593033,
+      "step": 100
+    },
+    {
+      "clip_ratio": 0.0006831295904703438,
+      "epoch": 0.0018842404738584953,
+      "grad_norm": 0.08466409146785736,
+      "kl": 0.016815185546875,
+      "learning_rate": 4.659822517785203e-06,
+      "loss": -0.0103,
+      "step": 101
+    },
+    {
+      "clip_ratio": 0.000901765946764499,
+      "epoch": 0.001902896320134322,
+      "grad_norm": 0.09180773049592972,
+      "kl": 0.017242431640625,
+      "learning_rate": 4.6520770500166165e-06,
+      "loss": -0.0105,
+      "step": 102
+    },
+    {
+      "clip_ratio": 0.0008878828666638583,
+      "completion_length": 771.9583435058594,
+      "epoch": 0.0019215521664101489,
+      "grad_norm": 0.11708436161279678,
+      "kl": 0.029144287109375,
+      "learning_rate": 4.644251774904487e-06,
+      "loss": 0.0148,
+      "num_tokens": 1188838.0,
+      "reward": 0.03181159868836403,
+      "reward_std": 0.02184708882123232,
+      "rewards/code_reward": 0.0018115942366421223,
+      "rewards/format_reward": 0.29999998211860657,
+      "step": 103
+    },
+    {
+      "clip_ratio": 0.0006120741018094122,
+      "epoch": 0.0019402080126859754,
+      "grad_norm": 0.10810239613056183,
+      "kl": 0.03369140625,
+      "learning_rate": 4.636347020781684e-06,
+      "loss": 0.0148,
+      "step": 104
+    },
+    {
+      "clip_ratio": 0.0008727995736990124,
+      "epoch": 0.0019588638589618022,
+      "grad_norm": 0.11280914396047592,
+      "kl": 0.0435791015625,
+      "learning_rate": 4.6283631193158605e-06,
+      "loss": 0.0145,
+      "step": 105
+    },
+    {
+      "clip_ratio": 0.0009058912692125887,
+      "completion_length": 914.5417175292969,
+      "epoch": 0.0019775197052376288,
+      "grad_norm": 0.07587506622076035,
+      "kl": 0.0174560546875,
+      "learning_rate": 4.620300405495532e-06,
+      "loss": 0.0523,
+      "num_tokens": 1222319.0,
+      "reward": 0.15707794576883316,
+      "reward_std": 0.2763161137700081,
+      "rewards/code_reward": 0.12624460831284523,
+      "rewards/format_reward": 0.3083333447575569,
+      "step": 106
+    },
+    {
+      "clip_ratio": 0.0009446362673770636,
+      "epoch": 0.0019961755515134553,
+      "grad_norm": 0.07568401843309402,
+      "kl": 0.01898193359375,
+      "learning_rate": 4.612159217616022e-06,
+      "loss": 0.0523,
+      "step": 107
+    },
+    {
+      "clip_ratio": 0.0011543052969500422,
+      "epoch": 0.0020148313977892823,
+      "grad_norm": 0.07626564055681229,
+      "kl": 0.02130126953125,
+      "learning_rate": 4.603939897265268e-06,
+      "loss": 0.0524,
+      "step": 108
+    },
+    {
+      "clip_ratio": 0.0008169686479959637,
+      "completion_length": 897.7500305175781,
+      "epoch": 0.002033487244065109,
+      "grad_norm": 0.08059542626142502,
+      "kl": 0.02227783203125,
+      "learning_rate": 4.595642789309492e-06,
+      "loss": 0.0107,
+      "num_tokens": 1259393.0,
+      "reward": 0.4133833199739456,
+      "reward_std": 0.4549361765384674,
+      "rewards/code_reward": 0.3917166441679001,
+      "rewards/format_reward": 0.21666666865348816,
+      "step": 109
+    },
+    {
+      "clip_ratio": 0.0010060299537144601,
+      "epoch": 0.0020521430903409354,
+      "grad_norm": 0.08048728853464127,
+      "kl": 0.0216064453125,
+      "learning_rate": 4.587268241878724e-06,
+      "loss": 0.0106,
+      "step": 110
+    },
+    {
+      "clip_ratio": 0.0012289943406358361,
+      "epoch": 0.0020707989366167624,
+      "grad_norm": 0.07647143304347992,
+      "kl": 0.019775390625,
+      "learning_rate": 4.578816606352205e-06,
+      "loss": 0.0107,
+      "step": 111
+    },
+    {
+      "clip_ratio": 0.0013880816695746034,
+      "completion_length": 1007.6667175292969,
+      "epoch": 0.002089454782892589,
+      "grad_norm": 0.07475873827934265,
+      "kl": 0.01617431640625,
+      "learning_rate": 4.570288237343632e-06,
+      "loss": 0.0233,
+      "num_tokens": 1298889.0,
+      "reward": 0.16264833509922028,
+      "reward_std": 0.22893914580345154,
+      "rewards/code_reward": 0.14723165333271027,
+      "rewards/format_reward": 0.15416666120290756,
+      "step": 112
+    },
+    {
+      "clip_ratio": 0.0013335386465769261,
+      "epoch": 0.0021081106291684156,
+      "grad_norm": 0.07221872359514236,
+      "kl": 0.01495361328125,
+      "learning_rate": 4.561683492686289e-06,
+      "loss": 0.0231,
+      "step": 113
+    },
+    {
+      "clip_ratio": 0.0011128429905511439,
+      "epoch": 0.0021267664754442426,
+      "grad_norm": 0.11232966184616089,
+      "kl": 0.013580322265625,
+      "learning_rate": 4.5530027334180285e-06,
+      "loss": 0.0232,
+      "step": 114
+    },
+    {
+      "clip_ratio": 0.0007180829998105764,
+      "completion_length": 760.375,
+      "epoch": 0.002145422321720069,
+      "grad_norm": 0.08500614017248154,
+      "kl": 0.0218505859375,
+      "learning_rate": 4.544246323766122e-06,
+      "loss": 0.0176,
+      "num_tokens": 1329738.0,
+      "reward": 0.42050473392009735,
+      "reward_std": 0.4651963710784912,
+      "rewards/code_reward": 0.37967143952846527,
+      "rewards/format_reward": 0.40833334624767303,
+      "step": 115
+    },
+    {
+      "clip_ratio": 0.0013084964302834123,
+      "epoch": 0.0021640781679958957,
+      "grad_norm": 0.09566804766654968,
+      "kl": 0.01910400390625,
+      "learning_rate": 4.535414631131983e-06,
+      "loss": 0.0174,
+      "step": 116
+    },
+    {
+      "clip_ratio": 0.0012152747949585319,
+      "epoch": 0.0021827340142717222,
+      "grad_norm": 0.09403965622186661,
+      "kl": 0.01824951171875,
+      "learning_rate": 4.526508026075746e-06,
+      "loss": 0.0175,
+      "step": 117
+    },
+    {
+      "clip_ratio": 0.00110370441689156,
+      "completion_length": 913.9583740234375,
+      "epoch": 0.0022013898605475492,
+      "grad_norm": 0.09698626399040222,
+      "kl": 0.022411346435546875,
+      "learning_rate": 4.517526882300721e-06,
+      "loss": -0.0229,
+      "num_tokens": 1372757.0,
+      "reward": 0.3174999728798866,
+      "reward_std": 0.27699745260179043,
+      "rewards/code_reward": 0.2916666567325592,
+      "rewards/format_reward": 0.25833334028720856,
+      "step": 118
+    },
+    {
+      "clip_ratio": 0.0013115269830450416,
+      "epoch": 0.002220045706823376,
+      "grad_norm": 0.09875179827213287,
+      "kl": 0.019748687744140625,
+      "learning_rate": 4.508471576637713e-06,
+      "loss": -0.0228,
+      "step": 119
+    },
+    {
+      "clip_ratio": 0.0011575823882594705,
+      "epoch": 0.0022387015530992023,
+      "grad_norm": 0.10077529400587082,
+      "kl": 0.01830291748046875,
+      "learning_rate": 4.499342489029211e-06,
+      "loss": -0.0231,
+      "step": 120
+    },
+    {
+      "clip_ratio": 0.0006889864162076265,
+      "completion_length": 498.3750305175781,
+      "epoch": 0.0022573573993750293,
+      "grad_norm": 0.11392726749181747,
+      "kl": 0.025848388671875,
+      "learning_rate": 4.490140002513449e-06,
+      "loss": 0.027,
+      "num_tokens": 1393730.0,
+      "reward": 0.9718478322029114,
+      "reward_std": 0.2110026255249977,
+      "rewards/code_reward": 0.9293478429317474,
+      "rewards/format_reward": 0.42499999701976776,
+      "step": 121
+    },
+    {
+      "clip_ratio": 0.0005561001598834991,
+      "epoch": 0.002276013245650856,
+      "grad_norm": 0.10952252149581909,
+      "kl": 0.025726318359375,
+      "learning_rate": 4.48086450320833e-06,
+      "loss": 0.0267,
+      "step": 122
+    },
+    {
+      "clip_ratio": 0.0006027935305610299,
+      "epoch": 0.0022946690919266825,
+      "grad_norm": 0.1085001602768898,
+      "kl": 0.0263671875,
+      "learning_rate": 4.4715163802952266e-06,
+      "loss": 0.0269,
+      "step": 123
+    },
+    {
+      "clip_ratio": 0.001191554736578837,
+      "completion_length": 732.75,
+      "epoch": 0.002313324938202509,
+      "grad_norm": 0.08821776509284973,
+      "kl": 0.031219482421875,
+      "learning_rate": 4.462096026002655e-06,
+      "loss": 0.0062,
+      "num_tokens": 1422824.0,
+      "reward": 0.03973765671253204,
+      "reward_std": 0.03781243972480297,
+      "rewards/code_reward": 0.00848765391856432,
+      "rewards/format_reward": 0.3125000074505806,
+      "step": 124
+    },
+    {
+      "clip_ratio": 0.001063963572960347,
+      "epoch": 0.002331980784478336,
+      "grad_norm": 0.09551744163036346,
+      "kl": 0.0391845703125,
+      "learning_rate": 4.4526038355898144e-06,
+      "loss": 0.0061,
+      "step": 125
+    },
+    {
+      "clip_ratio": 0.0010462789214216173,
+      "epoch": 0.0023506366307541626,
+      "grad_norm": 0.08740315586328506,
+      "kl": 0.03314208984375,
+      "learning_rate": 4.4430402073300035e-06,
+      "loss": 0.0058,
+      "step": 126
+    },
+    {
+      "clip_ratio": 0.0009691159357316792,
+      "completion_length": 1037.7500610351562,
+      "epoch": 0.002369292477029989,
+      "grad_norm": 0.07823047786951065,
+      "kl": 0.0126953125,
+      "learning_rate": 4.433405542493909e-06,
+      "loss": -0.0015,
+      "num_tokens": 1459130.0,
+      "reward": 0.6776870489120483,
+      "reward_std": 0.10291638784110546,
+      "rewards/code_reward": 0.6343537420034409,
+      "rewards/format_reward": 0.4333333447575569,
+      "step": 127
+    },
+    {
+      "clip_ratio": 0.001145587011706084,
+      "epoch": 0.002387948323305816,
+      "grad_norm": 0.07594677060842514,
+      "kl": 0.012725830078125,
+      "learning_rate": 4.4237002453327734e-06,
+      "loss": -0.0017,
+      "step": 128
+    },
+    {
+      "clip_ratio": 0.0008910182514227927,
+      "epoch": 0.0024066041695816427,
+      "grad_norm": 0.07640928775072098,
+      "kl": 0.0137939453125,
+      "learning_rate": 4.4139247230614245e-06,
+      "loss": -0.0019,
+      "step": 129
+    },
+    {
+      "clip_ratio": 0.0011884173727594316,
+      "completion_length": 1120.7916870117188,
+      "epoch": 0.0024252600158574692,
+      "grad_norm": 0.07030566036701202,
+      "kl": 0.014190673828125,
+      "learning_rate": 4.404079385841201e-06,
+      "loss": 0.0265,
+      "num_tokens": 1506729.0,
+      "reward": 0.7509399354457855,
+      "reward_std": 0.3766524940729141,
+      "rewards/code_reward": 0.7180232405662537,
+      "rewards/format_reward": 0.3291666656732559,
+      "step": 130
+    },
+    {
+      "clip_ratio": 0.0011372754233889282,
+      "epoch": 0.002443915862133296,
+      "grad_norm": 0.07563882321119308,
+      "kl": 0.015655517578125,
+      "learning_rate": 4.394164646762734e-06,
+      "loss": 0.0264,
+      "step": 131
+    },
+    {
+      "clip_ratio": 0.0009669371065683663,
+      "epoch": 0.002462571708409123,
+      "grad_norm": 0.07320787757635117,
+      "kl": 0.014190673828125,
+      "learning_rate": 4.384180921828618e-06,
+      "loss": 0.0263,
+      "step": 132
+    },
+    {
+      "clip_ratio": 0.0012862688163295388,
+      "completion_length": 733.4166870117188,
+      "epoch": 0.0024812275546849494,
+      "grad_norm": 0.09955161809921265,
+      "kl": 0.013763427734375,
+      "learning_rate": 4.374128629935955e-06,
+      "loss": 0.0072,
+      "num_tokens": 1537483.0,
+      "reward": 0.5765306204557419,
+      "reward_std": 0.14132879488170147,
+      "rewards/code_reward": 0.5348639450967312,
+      "rewards/format_reward": 0.4166666716337204,
+      "step": 133
+    },
+    {
+      "clip_ratio": 0.0008828279678709805,
+      "epoch": 0.002499883400960776,
+      "grad_norm": 0.10158013552427292,
+      "kl": 0.014129638671875,
+      "learning_rate": 4.364008192858781e-06,
+      "loss": 0.0072,
+      "step": 134
+    },
+    {
+      "clip_ratio": 0.0012895396212115884,
+      "epoch": 0.002518539247236603,
+      "grad_norm": 0.08935025334358215,
+      "kl": 0.01409912109375,
+      "learning_rate": 4.353820035230366e-06,
+      "loss": 0.007,
+      "step": 135
+    },
+    {
+      "clip_ratio": 0.000699867494404316,
+      "completion_length": 921.0833435058594,
+      "epoch": 0.0025371950935124295,
+      "grad_norm": 0.07964594662189484,
+      "kl": 0.00946044921875,
+      "learning_rate": 4.3435645845254e-06,
+      "loss": 0.0365,
+      "num_tokens": 1577169.0,
+      "reward": 0.04625000339001417,
+      "reward_std": 0.0623414209112525,
+      "rewards/code_reward": 0.02083333395421505,
+      "rewards/format_reward": 0.2541666626930237,
+      "step": 136
+    },
+    {
+      "clip_ratio": 0.0003799215774051845,
+      "epoch": 0.002555850939788256,
+      "grad_norm": 0.08275993913412094,
+      "kl": 0.010284423828125,
+      "learning_rate": 4.333242271042054e-06,
+      "loss": 0.0363,
+      "step": 137
+    },
+    {
+      "clip_ratio": 0.00039069188642315567,
+      "epoch": 0.002574506786064083,
+      "grad_norm": 0.08476851880550385,
+      "kl": 0.0107574462890625,
+      "learning_rate": 4.32285352788393e-06,
+      "loss": 0.0363,
+      "step": 138
+    },
+    {
+      "clip_ratio": 0.0006457642884925008,
+      "completion_length": 873.1250305175781,
+      "epoch": 0.0025931626323399096,
+      "grad_norm": 0.11129351705312729,
+      "kl": 0.015472412109375,
+      "learning_rate": 4.312398790941882e-06,
+      "loss": 0.0132,
+      "num_tokens": 1613100.0,
+      "reward": 0.2220833506435156,
+      "reward_std": 0.12337806262075901,
+      "rewards/code_reward": 0.17291666567325592,
+      "rewards/format_reward": 0.49166665971279144,
+      "step": 139
+    },
+    {
+      "clip_ratio": 0.0009015932446345687,
+      "epoch": 0.002611818478615736,
+      "grad_norm": 0.09786652773618698,
+      "kl": 0.01611328125,
+      "learning_rate": 4.301878498875735e-06,
+      "loss": 0.0132,
+      "step": 140
+    },
+    {
+      "clip_ratio": 0.000771370017901063,
+      "epoch": 0.0026304743248915627,
+      "grad_norm": 0.09219111502170563,
+      "kl": 0.017333984375,
+      "learning_rate": 4.291293093095873e-06,
+      "loss": 0.0128,
+      "step": 141
+    },
+    {
+      "clip_ratio": 0.00134377705398947,
+      "completion_length": 758.8333740234375,
+      "epoch": 0.0026491301711673897,
+      "grad_norm": 0.09211676567792892,
+      "kl": 0.048370361328125,
+      "learning_rate": 4.280643017744723e-06,
+      "loss": 0.0087,
+      "num_tokens": 1640192.0,
+      "reward": 0.4495016485452652,
+      "reward_std": 0.13479312881827354,
+      "rewards/code_reward": 0.3999183177947998,
+      "rewards/format_reward": 0.4958333224058151,
+      "step": 142
+    },
+    {
+      "clip_ratio": 0.0010106293557328172,
+      "epoch": 0.0026677860174432163,
+      "grad_norm": 0.0978454202413559,
+      "kl": 0.055938720703125,
+      "learning_rate": 4.269928719678117e-06,
+      "loss": 0.0087,
+      "step": 143
+    },
+    {
+      "clip_ratio": 0.0008834551408654079,
+      "epoch": 0.002686441863719043,
+      "grad_norm": 0.09563044458627701,
+      "kl": 0.056365966796875,
+      "learning_rate": 4.2591506484465426e-06,
+      "loss": 0.0087,
+      "step": 144
+    },
+    {
+      "clip_ratio": 0.0010417406738270074,
+      "completion_length": 811.8333740234375,
+      "epoch": 0.00270509770999487,
+      "grad_norm": 0.09586314111948013,
+      "kl": 0.0595703125,
+      "learning_rate": 4.248309256276283e-06,
+      "loss": 0.0134,
+      "num_tokens": 1674328.0,
+      "reward": 0.4281481206417084,
+      "reward_std": 0.351884625852108,
+      "rewards/code_reward": 0.39814815670251846,
+      "rewards/format_reward": 0.29999999701976776,
+      "step": 145
+    },
+    {
+      "clip_ratio": 0.0013275437522679567,
+      "epoch": 0.0027237535562706964,
+      "grad_norm": 0.08591298758983612,
+      "kl": 0.050048828125,
+      "learning_rate": 4.23740499805044e-06,
+      "loss": 0.0134,
+      "step": 146
+    },
+    {
+      "clip_ratio": 0.0016137410420924425,
+      "epoch": 0.002742409402546523,
+      "grad_norm": 0.08595118671655655,
+      "kl": 0.03662109375,
+      "learning_rate": 4.22643833128985e-06,
+      "loss": 0.0129,
+      "step": 147
+    },
+    {
+      "clip_ratio": 0.00041602051351219416,
+      "completion_length": 819.7083740234375,
+      "epoch": 0.0027610652488223495,
+      "grad_norm": 0.09021999686956406,
+      "kl": 0.03216552734375,
+      "learning_rate": 4.215409716133885e-06,
+      "loss": 0.0092,
+      "num_tokens": 1707417.0,
+      "reward": 0.42956283316016197,
+      "reward_std": 0.21959564834833145,
+      "rewards/code_reward": 0.38414610736072063,
+      "rewards/format_reward": 0.4541666805744171,
+      "step": 148
+    },
+    {
+      "clip_ratio": 0.0007702489383518696,
+      "epoch": 0.0027797210950981765,
+      "grad_norm": 0.08686275035142899,
+      "kl": 0.023193359375,
+      "learning_rate": 4.204319615321151e-06,
+      "loss": 0.0092,
+      "step": 149
+    },
+    {
+      "clip_ratio": 0.0006736835566698574,
+      "epoch": 0.002798376941374003,
+      "grad_norm": 0.08748073130846024,
+      "kl": 0.01898193359375,
+      "learning_rate": 4.193168494170065e-06,
+      "loss": 0.009,
+      "step": 150
+    },
+    {
+      "clip_ratio": 0.0005513402575161308,
+      "completion_length": 924.9166870117188,
+      "epoch": 0.0028170327876498296,
+      "grad_norm": 0.08234385401010513,
+      "kl": 0.008636474609375,
+      "learning_rate": 4.181956820559339e-06,
+      "loss": -0.0217,
+      "num_tokens": 1751527.0,
+      "reward": 0.5591950714588165,
+      "reward_std": 0.18445394933223724,
+      "rewards/code_reward": 0.5187783911824226,
+      "rewards/format_reward": 0.40416665375232697,
+      "step": 151
+    },
+    {
+      "clip_ratio": 0.0006712005124427378,
+      "epoch": 0.0028356886339256566,
+      "grad_norm": 0.0800977274775505,
+      "kl": 0.006988525390625,
+      "learning_rate": 4.170685064908342e-06,
+      "loss": -0.0219,
+      "step": 152
+    },
+    {
+      "clip_ratio": 0.0005624309997074306,
+      "epoch": 0.002854344480201483,
+      "grad_norm": 0.08224301040172577,
+      "kl": 0.0066986083984375,
+      "learning_rate": 4.159353700157365e-06,
+      "loss": -0.022,
+      "step": 153
+    },
+    {
+      "clip_ratio": 0.00046220331569202244,
+      "completion_length": 557.3333435058594,
+      "epoch": 0.0028730003264773097,
+      "grad_norm": 0.09233321249485016,
+      "kl": 0.028167724609375,
+      "learning_rate": 4.14796320174778e-06,
+      "loss": -0.013,
+      "num_tokens": 1781007.0,
+      "reward": 0.6308333575725555,
+      "reward_std": 0.449993371963501,
+      "rewards/code_reward": 0.5833333283662796,
+      "rewards/format_reward": 0.4750000089406967,
+      "step": 154
+    },
+    {
+      "clip_ratio": 0.0002166997583117336,
+      "epoch": 0.0028916561727531367,
+      "grad_norm": 0.0909167230129242,
+      "kl": 0.02734375,
+      "learning_rate": 4.136514047602087e-06,
+      "loss": -0.0133,
+      "step": 155
+    },
+    {
+      "clip_ratio": 0.00029903331596869975,
+      "epoch": 0.0029103120190289633,
+      "grad_norm": 0.09201790392398834,
+      "kl": 0.0225372314453125,
+      "learning_rate": 4.1250067181038635e-06,
+      "loss": -0.0133,
+      "step": 156
+    },
+    {
+      "clip_ratio": 0.000859909545397386,
+      "completion_length": 1166.7083740234375,
+      "epoch": 0.00292896786530479,
+      "grad_norm": 0.12645356357097626,
+      "kl": 0.0063934326171875,
+      "learning_rate": 4.113441696077608e-06,
+      "loss": 0.0052,
+      "num_tokens": 1827500.0,
+      "reward": 0.04448845051229,
+      "reward_std": 0.028481798246502876,
+      "rewards/code_reward": 0.009488449431955814,
+      "rewards/format_reward": 0.3499999940395355,
+      "step": 157
+    },
+    {
+      "clip_ratio": 0.0006146969681140035,
+      "epoch": 0.0029476237115806164,
+      "grad_norm": 0.09026872366666794,
+      "kl": 0.00640869140625,
+      "learning_rate": 4.101819466768484e-06,
+      "loss": 0.0051,
+      "step": 158
+    },
+    {
+      "clip_ratio": 0.0006830913771409541,
+      "epoch": 0.0029662795578564434,
+      "grad_norm": 0.09458412230014801,
+      "kl": 0.007568359375,
+      "learning_rate": 4.0901405178219535e-06,
+      "loss": 0.005,
+      "step": 159
+    },
+    {
+      "clip_ratio": 0.0009271332528442144,
+      "completion_length": 902.75,
+      "epoch": 0.00298493540413227,
+      "grad_norm": 0.09097299724817276,
+      "kl": 0.0082550048828125,
+      "learning_rate": 4.078405339263326e-06,
+      "loss": 0.0325,
+      "num_tokens": 1863974.0,
+      "reward": 0.3194986581802368,
+      "reward_std": 0.23043666034936905,
+      "rewards/code_reward": 0.28074865974485874,
+      "rewards/format_reward": 0.38750001788139343,
+      "step": 160
+    },
+    {
+      "clip_ratio": 0.001312960172072053,
+      "epoch": 0.0030035912504080965,
+      "grad_norm": 0.08883295208215714,
+      "kl": 0.00946044921875,
+      "learning_rate": 4.06661442347719e-06,
+      "loss": 0.0322,
+      "step": 161
+    },
+    {
+      "clip_ratio": 0.00119069468928501,
+      "epoch": 0.0030222470966839235,
+      "grad_norm": 0.08946386724710464,
+      "kl": 0.010009765625,
+      "learning_rate": 4.054768265186758e-06,
+      "loss": 0.0322,
+      "step": 162
+    },
+    {
+      "clip_ratio": 0.0016622182447463274,
+      "completion_length": 831.4583435058594,
+      "epoch": 0.00304090294295975,
+      "grad_norm": 0.07739204913377762,
+      "kl": 0.0092010498046875,
+      "learning_rate": 4.0428673614331036e-06,
+      "loss": -0.0206,
+      "num_tokens": 1901017.0,
+      "reward": 0.39075271785259247,
+      "reward_std": 0.4852019101381302,
+      "rewards/code_reward": 0.36116939783096313,
+      "rewards/format_reward": 0.2958333343267441,
+      "step": 163
+    },
+    {
+      "clip_ratio": 0.0012105648347642273,
+      "epoch": 0.0030595587892355766,
+      "grad_norm": 0.07764855027198792,
+      "kl": 0.00982666015625,
+      "learning_rate": 4.030912211554316e-06,
+      "loss": -0.0211,
+      "step": 164
+    },
+    {
+      "clip_ratio": 0.001361995586194098,
+      "epoch": 0.003078214635511403,
+      "grad_norm": 0.0787842720746994,
+      "kl": 0.0101318359375,
+      "learning_rate": 4.018903317164539e-06,
+      "loss": -0.0208,
+      "step": 165
+    },
+    {
+      "clip_ratio": 0.00047827488742768764,
+      "completion_length": 970.7916870117188,
+      "epoch": 0.00309687048178723,
+      "grad_norm": 0.09784507006406784,
+      "kl": 0.010040283203125,
+      "learning_rate": 4.006841182132932e-06,
+      "loss": 0.0032,
+      "num_tokens": 1933292.0,
+      "reward": 0.7487499713897705,
+      "reward_std": 0.4054149240255356,
+      "rewards/code_reward": 0.7083333432674408,
+      "rewards/format_reward": 0.40416666865348816,
+      "step": 166
+    },
+    {
+      "clip_ratio": 0.0005554754170589149,
+      "epoch": 0.0031155263280630567,
+      "grad_norm": 0.09923358261585236,
+      "kl": 0.01019287109375,
+      "learning_rate": 3.9947263125625195e-06,
+      "loss": 0.0032,
+      "step": 167
+    },
+    {
+      "clip_ratio": 0.00030202302150428295,
+      "epoch": 0.0031341821743388833,
+      "grad_norm": 0.09393054246902466,
+      "kl": 0.0109100341796875,
+      "learning_rate": 3.982559216768967e-06,
+      "loss": 0.0029,
+      "step": 168
+    },
+    {
+      "clip_ratio": 0.0007342632743529975,
+      "completion_length": 645.7500305175781,
+      "epoch": 0.0031528380206147103,
+      "grad_norm": 0.09612008929252625,
+      "kl": 0.01934814453125,
+      "learning_rate": 3.970340405259245e-06,
+      "loss": -0.0236,
+      "num_tokens": 1956074.0,
+      "reward": 0.6863030195236206,
+      "reward_std": 0.3077765703201294,
+      "rewards/code_reward": 0.6533863097429276,
+      "rewards/format_reward": 0.3291666731238365,
+      "step": 169
+    },
+    {
+      "clip_ratio": 0.0007756684790365398,
+      "epoch": 0.003171493866890537,
+      "grad_norm": 0.0964791551232338,
+      "kl": 0.02117919921875,
+      "learning_rate": 3.958070390710214e-06,
+      "loss": -0.0234,
+      "step": 170
+    },
+    {
+      "clip_ratio": 0.0010250475606881082,
+      "epoch": 0.0031901497131663634,
+      "grad_norm": 0.0937555730342865,
+      "kl": 0.02099609375,
+      "learning_rate": 3.945749687947109e-06,
+      "loss": -0.0239,
+      "step": 171
+    },
+    {
+      "clip_ratio": 0.001571565808262676,
+      "completion_length": 856.1666870117188,
+      "epoch": 0.0032088055594421904,
+      "grad_norm": 0.09850820899009705,
+      "kl": 0.014801025390625,
+      "learning_rate": 3.933378813921942e-06,
+      "loss": 0.0194,
+      "num_tokens": 1987470.0,
+      "reward": 0.048796952702105045,
+      "reward_std": 0.04604018107056618,
+      "rewards/code_reward": 0.021713614463806152,
+      "rewards/format_reward": 0.2708333358168602,
+      "step": 172
+    },
+    {
+      "clip_ratio": 0.0017044495907612145,
+      "epoch": 0.003227461405718017,
+      "grad_norm": 0.10293813049793243,
+      "kl": 0.01483154296875,
+      "learning_rate": 3.920958287691811e-06,
+      "loss": 0.0193,
+      "step": 173
+    },
+    {
+      "clip_ratio": 0.0016478158067911863,
+      "epoch": 0.0032461172519938435,
+      "grad_norm": 0.09644638746976852,
+      "kl": 0.017425537109375,
+      "learning_rate": 3.908488630397121e-06,
+      "loss": 0.0189,
+      "step": 174
+    },
+    {
+      "clip_ratio": 0.0009133272105827928,
+      "completion_length": 797.375,
+      "epoch": 0.00326477309826967,
+      "grad_norm": 0.08966498076915741,
+      "kl": 0.018890380859375,
+      "learning_rate": 3.8959703652397175e-06,
+      "loss": 0.0182,
+      "num_tokens": 2022255.0,
+      "reward": 0.4206944704055786,
+      "reward_std": 0.16624127328395844,
+      "rewards/code_reward": 0.3861111178994179,
+      "rewards/format_reward": 0.34583333134651184,
+      "step": 175
+    },
+    {
+      "clip_ratio": 0.0008382981468457729,
+      "epoch": 0.003283428944545497,
+      "grad_norm": 0.09205059707164764,
+      "kl": 0.019439697265625,
+      "learning_rate": 3.883404017460935e-06,
+      "loss": 0.0181,
+      "step": 176
+    },
+    {
+      "clip_ratio": 0.000650079753540922,
+      "epoch": 0.0033020847908213236,
+      "grad_norm": 0.10440210998058319,
+      "kl": 0.0201416015625,
+      "learning_rate": 3.870790114319559e-06,
+      "loss": 0.0182,
+      "step": 177
+    },
+    {
+      "clip_ratio": 0.00037280093238223344,
+      "completion_length": 798.9166870117188,
+      "epoch": 0.00332074063709715,
+      "grad_norm": 0.09877095371484756,
+      "kl": 0.0088043212890625,
+      "learning_rate": 3.858129185069701e-06,
+      "loss": 0.001,
+      "num_tokens": 2058181.0,
+      "reward": 0.6779166460037231,
+      "reward_std": 0.4303756207227707,
+      "rewards/code_reward": 0.625,
+      "rewards/format_reward": 0.5291666686534882,
+      "step": 178
+    },
+    {
+      "clip_ratio": 0.0005341586947906762,
+      "epoch": 0.003339396483372977,
+      "grad_norm": 0.09430207312107086,
+      "kl": 0.0088958740234375,
+      "learning_rate": 3.845421760938597e-06,
+      "loss": 0.0011,
+      "step": 179
+    },
+    {
+      "clip_ratio": 0.00040165129757951945,
+      "epoch": 0.0033580523296488037,
+      "grad_norm": 0.08880569040775299,
+      "kl": 0.0091552734375,
+      "learning_rate": 3.832668375104312e-06,
+      "loss": 0.0008,
+      "step": 180
+    },
+    {
+      "clip_ratio": 0.0010204909485764802,
+      "completion_length": 789.0833435058594,
+      "epoch": 0.0033767081759246303,
+      "grad_norm": 0.0856408104300499,
+      "kl": 0.0240478515625,
+      "learning_rate": 3.8198695626733725e-06,
+      "loss": 0.0168,
+      "num_tokens": 2085147.0,
+      "reward": 0.0904554259032011,
+      "reward_std": 0.16545533016324043,
+      "rewards/code_reward": 0.050872091203927994,
+      "rewards/format_reward": 0.3958333283662796,
+      "step": 181
+    },
+    {
+      "clip_ratio": 0.0013127815909683704,
+      "epoch": 0.003395364022200457,
+      "grad_norm": 0.08718743920326233,
+      "kl": 0.02423095703125,
+      "learning_rate": 3.8070258606583156e-06,
+      "loss": 0.0167,
+      "step": 182
+    },
+    {
+      "clip_ratio": 0.0010590656311251223,
+      "epoch": 0.003414019868476284,
+      "grad_norm": 0.08608557283878326,
+      "kl": 0.02398681640625,
+      "learning_rate": 3.7941378079551544e-06,
+      "loss": 0.0163,
+      "step": 183
+    },
+    {
+      "clip_ratio": 0.00044808804523199797,
+      "completion_length": 838.3333435058594,
+      "epoch": 0.0034326757147521104,
+      "grad_norm": 0.08882498741149902,
+      "kl": 0.04205322265625,
+      "learning_rate": 3.7812059453207677e-06,
+      "loss": -0.0138,
+      "num_tokens": 2118659.0,
+      "reward": 0.5362499505281448,
+      "reward_std": 0.020687240175902843,
+      "rewards/code_reward": 0.5,
+      "rewards/format_reward": 0.36249999701976776,
+      "step": 184
+    },
+    {
+      "clip_ratio": 0.0009204872185364366,
+      "epoch": 0.003451331561027937,
+      "grad_norm": 0.08916648477315903,
+      "kl": 0.044921875,
+      "learning_rate": 3.768230815350213e-06,
+      "loss": -0.0138,
+      "step": 185
+    },
+    {
+      "clip_ratio": 0.0004569297598209232,
+      "epoch": 0.003469987407303764,
+      "grad_norm": 0.08778577297925949,
+      "kl": 0.0452880859375,
+      "learning_rate": 3.7552129624539557e-06,
+      "loss": -0.0139,
+      "step": 186
+    },
+    {
+      "clip_ratio": 0.0006422713631764054,
+      "completion_length": 820.6666870117188,
+      "epoch": 0.0034886432535795905,
+      "grad_norm": 0.07521483302116394,
+      "kl": 0.02374267578125,
+      "learning_rate": 3.7421529328350316e-06,
+      "loss": -0.0011,
+      "num_tokens": 2154951.0,
+      "reward": 0.3243994116783142,
+      "reward_std": 0.16106001287698746,
+      "rewards/code_reward": 0.26398275047540665,
+      "rewards/format_reward": 0.6041666567325592,
+      "step": 187
+    },
+    {
+      "clip_ratio": 0.0007444258662872016,
+      "epoch": 0.003507299099855417,
+      "grad_norm": 0.07639120519161224,
+      "kl": 0.022552490234375,
+      "learning_rate": 3.7290512744661274e-06,
+      "loss": -0.0014,
+      "step": 188
+    },
+    {
+      "clip_ratio": 0.0008128004556056112,
+      "epoch": 0.003525954946131244,
+      "grad_norm": 0.07584098726511002,
+      "kl": 0.02203369140625,
+      "learning_rate": 3.715908537066589e-06,
+      "loss": -0.0016,
+      "step": 189
+    },
+    {
+      "clip_ratio": 0.0006247111014090478,
+      "completion_length": 782.2500610351562,
+      "epoch": 0.0035446107924070706,
+      "grad_norm": 0.09555865079164505,
+      "kl": 0.02532958984375,
+      "learning_rate": 3.7027252720793538e-06,
+      "loss": -0.0372,
+      "num_tokens": 2188473.0,
+      "reward": 0.640339195728302,
+      "reward_std": 0.48519615828990936,
+      "rewards/code_reward": 0.5903392136096954,
+      "rewards/format_reward": 0.5,
+      "step": 190
+    },
+    {
+      "clip_ratio": 0.0004575684142764658,
+      "epoch": 0.003563266638682897,
+      "grad_norm": 0.09690264612436295,
+      "kl": 0.0252685546875,
+      "learning_rate": 3.689502032647817e-06,
+      "loss": -0.0374,
+      "step": 191
+    },
+    {
+      "clip_ratio": 0.0003101055626757443,
+      "epoch": 0.0035819224849587238,
+      "grad_norm": 0.08903127908706665,
+      "kl": 0.02032470703125,
+      "learning_rate": 3.6762393735926245e-06,
+      "loss": -0.0377,
+      "step": 192
+    },
+    {
+      "clip_ratio": 0.00029177512624301016,
+      "completion_length": 791.9583435058594,
+      "epoch": 0.0036005783312345508,
+      "grad_norm": 0.10319360345602036,
+      "kl": 0.0065460205078125,
+      "learning_rate": 3.6629378513883852e-06,
+      "loss": 0.0565,
+      "num_tokens": 2221952.0,
+      "reward": 0.5221759006381035,
+      "reward_std": 0.40614573657512665,
+      "rewards/code_reward": 0.4759259298443794,
+      "rewards/format_reward": 0.4624999910593033,
+      "step": 193
+    },
+    {
+      "clip_ratio": 0.0005471540498547256,
+      "epoch": 0.0036192341775103773,
+      "grad_norm": 0.07854603230953217,
+      "kl": 0.00677490234375,
+      "learning_rate": 3.6495980241403307e-06,
+      "loss": 0.0563,
+      "step": 194
+    },
+    {
+      "clip_ratio": 0.0007658963440917432,
+      "epoch": 0.003637890023786204,
+      "grad_norm": 0.1044902577996254,
+      "kl": 0.0064697265625,
+      "learning_rate": 3.636220451560896e-06,
+      "loss": 0.0565,
+      "step": 195
+    },
+    {
+      "clip_ratio": 0.001431856129784137,
+      "completion_length": 845.7916870117188,
+      "epoch": 0.003656545870062031,
+      "grad_norm": 0.08788370341062546,
+      "kl": 0.015106201171875,
+      "learning_rate": 3.622805694946235e-06,
+      "loss": 0.0278,
+      "num_tokens": 2256867.0,
+      "reward": 0.55375000461936,
+      "reward_std": 0.04919072613120079,
+      "rewards/code_reward": 0.5,
+      "rewards/format_reward": 0.5374999940395355,
+      "step": 196
+    },
+    {
+      "clip_ratio": 0.0012035187100991607,
+      "epoch": 0.0036752017163378574,
+      "grad_norm": 0.08841948956251144,
+      "kl": 0.014556884765625,
+      "learning_rate": 3.609354317152667e-06,
+      "loss": 0.028,
+      "step": 197
+    },
+    {
+      "clip_ratio": 0.0011701789626386017,
+      "epoch": 0.003693857562613684,
+      "grad_norm": 0.08651836961507797,
+      "kl": 0.015625,
+      "learning_rate": 3.595866882573063e-06,
+      "loss": 0.0275,
+      "step": 198
+    },
+    {
+      "clip_ratio": 0.0006608768162550405,
+      "completion_length": 812.9583435058594,
+      "epoch": 0.0037125134088895105,
+      "grad_norm": 0.09646130353212357,
+      "kl": 0.010162353515625,
+      "learning_rate": 3.5823439571131675e-06,
+      "loss": -0.0052,
+      "num_tokens": 2286290.0,
+      "reward": 0.09254807978868484,
+      "reward_std": 0.06599834933876991,
+      "rewards/code_reward": 0.030048077926039696,
+      "rewards/format_reward": 0.625,
+      "step": 199
+    },
+    {
+      "clip_ratio": 0.000630842725513503,
+      "epoch": 0.0037311692551653375,
+      "grad_norm": 0.09436467289924622,
+      "kl": 0.0110015869140625,
+      "learning_rate": 3.5687861081678477e-06,
+      "loss": -0.0052,
+      "step": 200
+    },
+    {
+      "clip_ratio": 0.0007917208422441036,
+      "epoch": 0.003749825101441164,
+      "grad_norm": 0.09504929929971695,
+      "kl": 0.012542724609375,
+      "learning_rate": 3.555193904597291e-06,
+      "loss": -0.0053,
+      "step": 201
+    },
+    {
+      "clip_ratio": 0.0005567904736381024,
+      "completion_length": 846.1250305175781,
+      "epoch": 0.0037684809477169907,
+      "grad_norm": 0.08655253797769547,
+      "kl": 0.0111083984375,
+      "learning_rate": 3.541567916703138e-06,
+      "loss": 0.0168,
+      "num_tokens": 2316833.0,
+      "reward": 0.46897655725479126,
+      "reward_std": 0.23874285072088242,
+      "rewards/code_reward": 0.4368932172656059,
+      "rewards/format_reward": 0.32083334028720856,
+      "step": 202
+    },
+    {
+      "clip_ratio": 0.0003927100624423474,
+      "epoch": 0.0037871367939928176,
+      "grad_norm": 0.08548427373170853,
+      "kl": 0.01123046875,
+      "learning_rate": 3.5279087162045517e-06,
+      "loss": 0.0168,
+      "step": 203
+    },
+    {
+      "clip_ratio": 0.0006460061413235962,
+      "epoch": 0.003805792640268644,
+      "grad_norm": 0.08546783775091171,
+      "kl": 0.011199951171875,
+      "learning_rate": 3.5142168762142265e-06,
+      "loss": 0.0165,
+      "step": 204
+    },
+    {
+      "clip_ratio": 0.0005783264059573412,
+      "completion_length": 671.1666870117188,
+      "epoch": 0.0038244484865444708,
+      "grad_norm": 0.2734091579914093,
+      "kl": 0.020843505859375,
+      "learning_rate": 3.500492971214347e-06,
+      "loss": 0.008,
+      "num_tokens": 2344797.0,
+      "reward": 0.5388242453336716,
+      "reward_std": 0.3791409581899643,
+      "rewards/code_reward": 0.4925742968916893,
+      "rewards/format_reward": 0.4625000059604645,
+      "step": 205
+    },
+    {
+      "clip_ratio": 0.0003238898643758148,
+      "epoch": 0.0038431043328202978,
+      "grad_norm": 0.08638158440589905,
+      "kl": 0.021759033203125,
+      "learning_rate": 3.48673757703248e-06,
+      "loss": 0.0078,
+      "step": 206
+    },
+    {
+      "clip_ratio": 0.00025367678608745337,
+      "epoch": 0.0038617601790961243,
+      "grad_norm": 0.08271009474992752,
+      "kl": 0.02239990234375,
+      "learning_rate": 3.472951270817418e-06,
+      "loss": 0.0077,
+      "step": 207
+    },
+    {
+      "clip_ratio": 0.0011410006263758987,
+      "completion_length": 863.6250610351562,
+      "epoch": 0.003880416025371951,
+      "grad_norm": 0.0871393084526062,
+      "kl": 0.009246826171875,
+      "learning_rate": 3.4591346310149578e-06,
+      "loss": 0.0031,
+      "num_tokens": 2380560.0,
+      "reward": 0.07268582284450531,
+      "reward_std": 0.057141270488500595,
+      "rewards/code_reward": 0.028519157320261,
+      "rewards/format_reward": 0.4416666626930237,
+      "step": 208
+    },
+    {
+      "clip_ratio": 0.0012201684294268489,
+      "epoch": 0.0038990718716477774,
+      "grad_norm": 0.08084117621183395,
+      "kl": 0.00958251953125,
+      "learning_rate": 3.445288237343632e-06,
+      "loss": 0.0031,
+      "step": 209
+    },
+    {
+      "clip_ratio": 0.0008913867350202054,
+      "epoch": 0.0039177277179236044,
+      "grad_norm": 0.08293391019105911,
+      "kl": 0.009307861328125,
+      "learning_rate": 3.4314126707703895e-06,
+      "loss": 0.003,
+      "step": 210
+    },
+    {
+      "clip_ratio": 0.0013628143933601677,
+      "completion_length": 695.0000305175781,
+      "epoch": 0.003936383564199431,
+      "grad_norm": 0.10922668874263763,
+      "kl": 0.0301513671875,
+      "learning_rate": 3.4175085134862128e-06,
+      "loss": 0.0343,
+      "num_tokens": 2409312.0,
+      "reward": 0.2725326791405678,
+      "reward_std": 0.31870152056217194,
+      "rewards/code_reward": 0.24836601875722408,
+      "rewards/format_reward": 0.24166666716337204,
+      "step": 211
+    },
+    {
+      "clip_ratio": 0.0010863369097933173,
+      "epoch": 0.0039550394104752576,
+      "grad_norm": 0.09670348465442657,
+      "kl": 0.024658203125,
+      "learning_rate": 3.4035763488816953e-06,
+      "loss": 0.0341,
+      "step": 212
+    },
+    {
+      "clip_ratio": 0.0009245816036127508,
+      "epoch": 0.003973695256751084,
+      "grad_norm": 0.0888768658041954,
+      "kl": 0.02496337890625,
+      "learning_rate": 3.3896167615225594e-06,
+      "loss": 0.0339,
+      "step": 213
+    },
+    {
+      "clip_ratio": 0.0007956430781632662,
+      "completion_length": 803.4583435058594,
+      "epoch": 0.003992351103026911,
+      "grad_norm": 0.08351018279790878,
+      "kl": 0.010467529296875,
+      "learning_rate": 3.375630337125133e-06,
+      "loss": -0.0005,
+      "num_tokens": 2439047.0,
+      "reward": 0.2963353507220745,
+      "reward_std": 0.2816299609839916,
+      "rewards/code_reward": 0.25050200801342726,
+      "rewards/format_reward": 0.4583333432674408,
+      "step": 214
+    },
+    {
+      "clip_ratio": 0.0009151631966233253,
+      "epoch": 0.004011006949302738,
+      "grad_norm": 0.0963514894247055,
+      "kl": 0.010101318359375,
+      "learning_rate": 3.361617662531772e-06,
+      "loss": -0.0006,
+      "step": 215
+    },
+    {
+      "clip_ratio": 0.0005119523120811209,
+      "epoch": 0.004029662795578565,
+      "grad_norm": 0.08659033477306366,
+      "kl": 0.0107421875,
+      "learning_rate": 3.347579325686237e-06,
+      "loss": -0.0011,
+      "step": 216
+    },
+    {
+      "clip_ratio": 0.0005694587889593095,
+      "completion_length": 548.8750305175781,
+      "epoch": 0.004048318641854391,
+      "grad_norm": 0.09764964878559113,
+      "kl": 0.015594482421875,
+      "learning_rate": 3.333515915609027e-06,
+      "loss": 0.0127,
+      "num_tokens": 2463224.0,
+      "reward": 0.8187499940395355,
+      "reward_std": 0.4159911423921585,
+      "rewards/code_reward": 0.75,
+      "rewards/format_reward": 0.6875,
+      "step": 217
+    },
+    {
+      "clip_ratio": 0.0006374841468641534,
+      "epoch": 0.004066974488130218,
+      "grad_norm": 0.09726841002702713,
+      "kl": 0.01580810546875,
+      "learning_rate": 3.3194280223726616e-06,
+      "loss": 0.0127,
+      "step": 218
+    },
+    {
+      "clip_ratio": 0.0008805101679172367,
+      "epoch": 0.004085630334406044,
+      "grad_norm": 0.09693842381238937,
+      "kl": 0.016326904296875,
+      "learning_rate": 3.305316237076927e-06,
+      "loss": 0.0126,
+      "step": 219
+    },
+    {
+      "clip_ratio": 0.001065359654603526,
+      "completion_length": 682.2083435058594,
+      "epoch": 0.004104286180681871,
+      "grad_norm": 0.11235346645116806,
+      "kl": 0.00555419921875,
+      "learning_rate": 3.291181151824071e-06,
+      "loss": 0.0219,
+      "num_tokens": 2487829.0,
+      "reward": 0.851570725440979,
+      "reward_std": 0.13773459196090698,
+      "rewards/code_reward": 0.7932373583316803,
+      "rewards/format_reward": 0.5833333134651184,
+      "step": 220
+    },
+    {
+      "clip_ratio": 0.0010035349405370653,
+      "epoch": 0.0041229420269576975,
+      "grad_norm": 0.10121559351682663,
+      "kl": 0.0056304931640625,
+      "learning_rate": 3.27702335969396e-06,
+      "loss": 0.0218,
+      "step": 221
+    },
+    {
+      "clip_ratio": 0.0007600956014357507,
+      "epoch": 0.004141597873233525,
+      "grad_norm": 0.09749292582273483,
+      "kl": 0.0055694580078125,
+      "learning_rate": 3.2628434547191985e-06,
+      "loss": 0.0217,
+      "step": 222
+    },
+    {
+      "clip_ratio": 0.00039983812894206494,
+      "completion_length": 822.5416870117188,
+      "epoch": 0.0041602537195093514,
+      "grad_norm": 0.08266626298427582,
+      "kl": 0.0115966796875,
+      "learning_rate": 3.2486420318601973e-06,
+      "loss": -0.0072,
+      "num_tokens": 2524514.0,
+      "reward": 0.765346884727478,
+      "reward_std": 0.41189485788345337,
+      "rewards/code_reward": 0.6945134997367859,
+      "rewards/format_reward": 0.7083333283662796,
+      "step": 223
+    },
+    {
+      "clip_ratio": 0.00040171146974898875,
+      "epoch": 0.004178909565785178,
+      "grad_norm": 0.08238179981708527,
+      "kl": 0.0108184814453125,
+      "learning_rate": 3.2344196869802187e-06,
+      "loss": -0.0073,
+      "step": 224
+    },
+    {
+      "clip_ratio": 0.0007332674576900899,
+      "epoch": 0.0041975654120610046,
+      "grad_norm": 0.08556170016527176,
+      "kl": 0.0115203857421875,
+      "learning_rate": 3.2201770168203694e-06,
+      "loss": -0.0072,
+      "step": 225
+    },
+    {
+      "clip_ratio": 0.0010465582308825105,
+      "completion_length": 897.4583435058594,
+      "epoch": 0.004216221258336831,
+      "grad_norm": 0.0795939639210701,
+      "kl": 0.010223388671875,
+      "learning_rate": 3.205914618974563e-06,
+      "loss": 0.0444,
+      "num_tokens": 2561713.0,
+      "reward": 0.5436052978038788,
+      "reward_std": 0.36193977296352386,
+      "rewards/code_reward": 0.49235527217388153,
+      "rewards/format_reward": 0.5125000178813934,
+      "step": 226
+    },
+    {
+      "clip_ratio": 0.001006987877190113,
+      "epoch": 0.004234877104612658,
+      "grad_norm": 0.0830455869436264,
+      "kl": 0.00970458984375,
+      "learning_rate": 3.1916330918644496e-06,
+      "loss": 0.0443,
+      "step": 227
+    },
+    {
+      "clip_ratio": 0.0006825106975156814,
+      "epoch": 0.004253532950888485,
+      "grad_norm": 0.07931126654148102,
+      "kl": 0.009521484375,
+      "learning_rate": 3.177333034714303e-06,
+      "loss": 0.0445,
+      "step": 228
+    },
+    {
+      "clip_ratio": 0.0010472153662703931,
+      "completion_length": 1060.8333740234375,
+      "epoch": 0.004272188797164312,
+      "grad_norm": 0.08189205825328827,
+      "kl": 0.01904296875,
+      "learning_rate": 3.1630150475258813e-06,
+      "loss": 0.0354,
+      "num_tokens": 2602557.0,
+      "reward": 0.5575000084936619,
+      "reward_std": 0.09106174670159817,
+      "rewards/code_reward": 0.520833333954215,
+      "rewards/format_reward": 0.36666667461395264,
+      "step": 229
+    },
+    {
+      "clip_ratio": 0.0012223481899127364,
+      "epoch": 0.004290844643440138,
+      "grad_norm": 0.07629529386758804,
+      "kl": 0.01898193359375,
+      "learning_rate": 3.148679731053252e-06,
+      "loss": 0.0355,
+      "step": 230
+    },
+    {
+      "clip_ratio": 0.0009530899405945092,
+      "epoch": 0.004309500489715965,
+      "grad_norm": 0.07555654644966125,
+      "kl": 0.01959228515625,
+      "learning_rate": 3.1343276867775805e-06,
+      "loss": 0.0352,
+      "step": 231
+    },
+    {
+      "clip_ratio": 0.000721571734175086,
+      "completion_length": 631.0,
+      "epoch": 0.004328156335991791,
+      "grad_norm": 0.08854309469461441,
+      "kl": 0.01287841796875,
+      "learning_rate": 3.1199595168819043e-06,
+      "loss": -0.014,
+      "num_tokens": 2627097.0,
+      "reward": 0.7598794400691986,
+      "reward_std": 0.4630916863679886,
+      "rewards/code_reward": 0.6998794674873352,
+      "rewards/format_reward": 0.6000000238418579,
+      "step": 232
+    },
+    {
+      "clip_ratio": 0.0004376633296487853,
+      "epoch": 0.004346812182267618,
+      "grad_norm": 0.1049780622124672,
+      "kl": 0.01348876953125,
+      "learning_rate": 3.105575824225852e-06,
+      "loss": -0.0143,
+      "step": 233
+    },
+    {
+      "clip_ratio": 0.00047352669935207814,
+      "epoch": 0.0043654680285434445,
+      "grad_norm": 0.08706912398338318,
+      "kl": 0.01336669921875,
+      "learning_rate": 3.091177212320363e-06,
+      "loss": -0.0145,
+      "step": 234
+    },
+    {
+      "clip_ratio": 0.00033213794813491404,
+      "completion_length": 1132.4166870117188,
+      "epoch": 0.004384123874819272,
+      "grad_norm": 0.08825133740901947,
+      "kl": 0.01216888427734375,
+      "learning_rate": 3.0767642853023538e-06,
+      "loss": -0.0049,
+      "num_tokens": 2672143.0,
+      "reward": 0.09307292103767395,
+      "reward_std": 0.049834271892905235,
+      "rewards/code_reward": 0.039322917349636555,
+      "rewards/format_reward": 0.5375000089406967,
+      "step": 235
+    },
+    {
+      "clip_ratio": 0.0003840103163383901,
+      "epoch": 0.0044027797210950985,
+      "grad_norm": 0.09255920350551605,
+      "kl": 0.01175689697265625,
+      "learning_rate": 3.062337647909376e-06,
+      "loss": -0.0051,
+      "step": 236
+    },
+    {
+      "clip_ratio": 0.00038397236494347453,
+      "epoch": 0.004421435567370925,
+      "grad_norm": 0.08827126771211624,
+      "kl": 0.010589599609375,
+      "learning_rate": 3.04789790545424e-06,
+      "loss": -0.0052,
+      "step": 237
+    },
+    {
+      "clip_ratio": 0.0007554197800345719,
+      "completion_length": 795.25,
+      "epoch": 0.004440091413646752,
+      "grad_norm": 0.09881194680929184,
+      "kl": 0.02008056640625,
+      "learning_rate": 3.033445663799621e-06,
+      "loss": 0.0047,
+      "num_tokens": 2704009.0,
+      "reward": 0.5524518974125385,
+      "reward_std": 0.056896304711699486,
+      "rewards/code_reward": 0.5012019231216982,
+      "rewards/format_reward": 0.5125000029802322,
+      "step": 238
+    },
+    {
+      "clip_ratio": 0.0010150580055778846,
+      "epoch": 0.004458747259922578,
+      "grad_norm": 0.09783502668142319,
+      "kl": 0.021484375,
+      "learning_rate": 3.018981529332633e-06,
+      "loss": 0.0046,
+      "step": 239
+    },
+    {
+      "clip_ratio": 0.0011725955409929156,
+      "epoch": 0.004477403106198405,
+      "grad_norm": 0.09466077387332916,
+      "kl": 0.0211181640625,
+      "learning_rate": 3.00450610893939e-06,
+      "loss": 0.0045,
+      "step": 240
+    },
+    {
+      "clip_ratio": 0.0011200214212294668,
+      "completion_length": 709.3750152587891,
+      "epoch": 0.004496058952474231,
+      "grad_norm": 0.11234831809997559,
+      "kl": 0.014251708984375,
+      "learning_rate": 2.9900200099795396e-06,
+      "loss": -0.001,
+      "num_tokens": 2735914.0,
+      "reward": 0.33983870036900043,
+      "reward_std": 0.2842044336721301,
+      "rewards/code_reward": 0.29233869910240173,
+      "rewards/format_reward": 0.4750000089406967,
+      "step": 241
+    },
+    {
+      "clip_ratio": 0.0012173219583928585,
+      "epoch": 0.004514714798750059,
+      "grad_norm": 0.10929913818836212,
+      "kl": 0.015838623046875,
+      "learning_rate": 2.9755238402607826e-06,
+      "loss": -0.0009,
+      "step": 242
+    },
+    {
+      "clip_ratio": 0.0010618427768349648,
+      "epoch": 0.004533370645025885,
+      "grad_norm": 0.1097099706530571,
+      "kl": 0.017913818359375,
+      "learning_rate": 2.961018208013367e-06,
+      "loss": -0.0014,
+      "step": 243
+    },
+    {
+      "clip_ratio": 0.0015606118831783533,
+      "completion_length": 740.5000305175781,
+      "epoch": 0.004552026491301712,
+      "grad_norm": 0.1023816242814064,
+      "kl": 0.010467529296875,
+      "learning_rate": 2.9465037218645694e-06,
+      "loss": -0.0155,
+      "num_tokens": 2764042.0,
+      "reward": 0.062079211696982384,
+      "reward_std": 0.04148745723068714,
+      "rewards/code_reward": 0.000412541237892583,
+      "rewards/format_reward": 0.6166666448116302,
+      "step": 244
+    },
+    {
+      "clip_ratio": 0.0012834252556785941,
+      "epoch": 0.004570682337577538,
+      "grad_norm": 0.09794943034648895,
+      "kl": 0.01129150390625,
+      "learning_rate": 2.9319809908131604e-06,
+      "loss": -0.0154,
+      "step": 245
+    },
+    {
+      "clip_ratio": 0.0014966255985200405,
+      "epoch": 0.004589338183853365,
+      "grad_norm": 0.09920386970043182,
+      "kl": 0.011505126953125,
+      "learning_rate": 2.917450624203847e-06,
+      "loss": -0.0159,
+      "step": 246
+    },
+    {
+      "clip_ratio": 0.0009015223768074065,
+      "completion_length": 675.5416870117188,
+      "epoch": 0.0046079940301291915,
+      "grad_norm": 0.08201450854539871,
+      "kl": 0.02886962890625,
+      "learning_rate": 2.9029132317017118e-06,
+      "loss": 0.0237,
+      "num_tokens": 2792831.0,
+      "reward": 0.4770751856267452,
+      "reward_std": 0.21596147678792477,
+      "rewards/code_reward": 0.41707515716552734,
+      "rewards/format_reward": 0.5999999940395355,
+      "step": 247
+    },
+    {
+      "clip_ratio": 0.0009108328085858375,
+      "epoch": 0.004626649876405018,
+      "grad_norm": 0.0822167694568634,
+      "kl": 0.0335693359375,
+      "learning_rate": 2.888369423266629e-06,
+      "loss": 0.0241,
+      "step": 248
+    },
+    {
+      "clip_ratio": 0.0011067874147556722,
+      "epoch": 0.0046453057226808455,
+      "grad_norm": 0.08312199264764786,
+      "kl": 0.0335693359375,
+      "learning_rate": 2.8738198091276712e-06,
+      "loss": 0.0238,
+      "step": 249
+    },
+    {
+      "clip_ratio": 0.0007156841747928411,
+      "completion_length": 705.1666870117188,
+      "epoch": 0.004663961568956672,
+      "grad_norm": 0.09860284626483917,
+      "kl": 0.033416748046875,
+      "learning_rate": 2.859264999757509e-06,
+      "loss": 0.0451,
+      "num_tokens": 2822631.0,
+      "reward": 0.5245833247900009,
+      "reward_std": 0.4078162908554077,
+      "rewards/code_reward": 0.4583333358168602,
+      "rewards/format_reward": 0.6624999940395355,
+      "step": 250
+    },
+    {
+      "clip_ratio": 0.0007305167673621327,
+      "epoch": 0.004682617415232499,
+      "grad_norm": 0.10016681998968124,
+      "kl": 0.0286865234375,
+      "learning_rate": 2.8447056058467928e-06,
+      "loss": 0.0451,
+      "step": 251
+    },
+    {
+      "clip_ratio": 0.0007170947210397571,
+      "epoch": 0.004701273261508325,
+      "grad_norm": 0.09779990464448929,
+      "kl": 0.029632568359375,
+      "learning_rate": 2.830142238278531e-06,
+      "loss": 0.045,
+      "step": 252
+    },
+    {
+      "clip_ratio": 0.000481658848002553,
+      "completion_length": 590.2916870117188,
+      "epoch": 0.004719929107784152,
+      "grad_norm": 154.7192840576172,
+      "kl": 92.0189208984375,
+      "learning_rate": 2.81557550810246e-06,
+      "loss": 0.9582,
+      "num_tokens": 2852470.0,
+      "reward": 0.5958333238959312,
+      "reward_std": 0.3366524577140808,
+      "rewards/code_reward": 0.541666679084301,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 253
+    },
+    {
+      "clip_ratio": 0.0004959633661201224,
+      "epoch": 0.004738584954059978,
+      "grad_norm": 25.69236946105957,
+      "kl": 15.9564208984375,
+      "learning_rate": 2.8010060265094026e-06,
+      "loss": 0.1926,
+      "step": 254
+    },
+    {
+      "clip_ratio": 0.0010467918764334172,
+      "epoch": 0.004757240800335805,
+      "grad_norm": 2.245051383972168,
+      "kl": 1.70458984375,
+      "learning_rate": 2.786434404805629e-06,
+      "loss": 0.0514,
+      "step": 255
+    },
+    {
+      "clip_ratio": 0.0009520899475319311,
+      "completion_length": 900.1250305175781,
+      "epoch": 0.004775896646611632,
+      "grad_norm": 0.0822591632604599,
+      "kl": 0.01397705078125,
+      "learning_rate": 2.771861254387199e-06,
+      "loss": -0.0279,
+      "num_tokens": 2891485.0,
+      "reward": 0.3816172480583191,
+      "reward_std": 0.25385507196187973,
+      "rewards/code_reward": 0.3220338970422745,
+      "rewards/format_reward": 0.5958333313465118,
+      "step": 256
+    },
+    {
+      "clip_ratio": 0.001093231316190213,
+      "epoch": 0.004794552492887459,
+      "grad_norm": 0.08108334243297577,
+      "kl": 0.01324462890625,
+      "learning_rate": 2.7572871867143204e-06,
+      "loss": -0.0278,
+      "step": 257
+    },
+    {
+      "clip_ratio": 0.0013529035350074992,
+      "epoch": 0.004813208339163285,
+      "grad_norm": 0.07985206693410873,
+      "kl": 0.01263427734375,
+      "learning_rate": 2.742712813285681e-06,
+      "loss": -0.0277,
+      "step": 258
+    },
+    {
+      "clip_ratio": 0.001162135595222935,
+      "completion_length": 1206.5833740234375,
+      "epoch": 0.004831864185439112,
+      "grad_norm": 0.06389827281236649,
+      "kl": 0.0082855224609375,
+      "learning_rate": 2.7281387456128017e-06,
+      "loss": -0.0507,
+      "num_tokens": 2938767.0,
+      "reward": 0.10169872269034386,
+      "reward_std": 0.16118879988789558,
+      "rewards/code_reward": 0.051282052882015705,
+      "rewards/format_reward": 0.5041666701436043,
+      "step": 259
+    },
+    {
+      "clip_ratio": 0.0009780275577213615,
+      "epoch": 0.0048505200317149385,
+      "grad_norm": 0.06325838714838028,
+      "kl": 0.008026123046875,
+      "learning_rate": 2.7135655951943716e-06,
+      "loss": -0.0507,
+      "step": 260
+    },
+    {
+      "clip_ratio": 0.0008303393551614136,
+      "epoch": 0.004869175877990765,
+      "grad_norm": 0.06411907821893692,
+      "kl": 0.00754547119140625,
+      "learning_rate": 2.698993973490598e-06,
+      "loss": -0.0507,
+      "step": 261
+    },
+    {
+      "clip_ratio": 0.0009404766315128654,
+      "completion_length": 799.3333435058594,
+      "epoch": 0.004887831724266592,
+      "grad_norm": 0.07781538367271423,
+      "kl": 0.00923919677734375,
+      "learning_rate": 2.6844244918975416e-06,
+      "loss": 0.0603,
+      "num_tokens": 2969987.0,
+      "reward": 0.532083310186863,
+      "reward_std": 0.1838482804596424,
+      "rewards/code_reward": 0.4583333432674408,
+      "rewards/format_reward": 0.737500011920929,
+      "step": 262
+    },
+    {
+      "clip_ratio": 0.0007310473010875285,
+      "epoch": 0.004906487570542419,
+      "grad_norm": 0.08621031045913696,
+      "kl": 0.0091094970703125,
+      "learning_rate": 2.66985776172147e-06,
+      "loss": 0.0604,
+      "step": 263
+    },
+    {
+      "clip_ratio": 0.0009290947928093374,
+      "epoch": 0.004925143416818246,
+      "grad_norm": 0.07831931114196777,
+      "kl": 0.00958251953125,
+      "learning_rate": 2.6552943941532088e-06,
+      "loss": 0.0602,
+      "step": 264
+    },
+    {
+      "clip_ratio": 0.0010500443167984486,
+      "completion_length": 902.4167175292969,
+      "epoch": 0.004943799263094072,
+      "grad_norm": 0.09909704327583313,
+      "kl": 0.013946533203125,
+      "learning_rate": 2.6407350002424927e-06,
+      "loss": -0.0336,
+      "num_tokens": 3008697.0,
+      "reward": 0.30541668087244034,
+      "reward_std": 0.2829319443553686,
+      "rewards/code_reward": 0.25,
+      "rewards/format_reward": 0.5541666597127914,
+      "step": 265
+    },
+    {
+      "clip_ratio": 0.0010122012172359973,
+      "epoch": 0.004962455109369899,
+      "grad_norm": 0.07800783216953278,
+      "kl": 0.014617919921875,
+      "learning_rate": 2.626180190872329e-06,
+      "loss": -0.0339,
+      "step": 266
+    },
+    {
+      "clip_ratio": 0.0009586416417732835,
+      "epoch": 0.004981110955645725,
+      "grad_norm": 0.08270706981420517,
+      "kl": 0.013427734375,
+      "learning_rate": 2.611630576733372e-06,
+      "loss": -0.034,
+      "step": 267
+    },
+    {
+      "clip_ratio": 0.0006794014479964972,
+      "completion_length": 852.8750610351562,
+      "epoch": 0.004999766801921552,
+      "grad_norm": 0.0892496407032013,
+      "kl": 0.01239013671875,
+      "learning_rate": 2.5970867682982885e-06,
+      "loss": -0.0121,
+      "num_tokens": 3046734.0,
+      "reward": 0.6499094069004059,
+      "reward_std": 0.21050915122032166,
+      "rewards/code_reward": 0.5765760540962219,
+      "rewards/format_reward": 0.7333333194255829,
+      "step": 268
+    },
+    {
+      "clip_ratio": 0.0004855144943576306,
+      "epoch": 0.005018422648197379,
+      "grad_norm": 0.08327967673540115,
+      "kl": 0.011993408203125,
+      "learning_rate": 2.582549375796154e-06,
+      "loss": -0.0124,
+      "step": 269
+    },
+    {
+      "clip_ratio": 0.000626907596597448,
+      "epoch": 0.005037078494473206,
+      "grad_norm": 0.08495430648326874,
+      "kl": 0.0126953125,
+      "learning_rate": 2.568019009186841e-06,
+      "loss": -0.0121,
+      "step": 270
+    },
+    {
+      "clip_ratio": 0.0007202730048447847,
+      "completion_length": 977.8333435058594,
+      "epoch": 0.005055734340749032,
+      "grad_norm": 0.09023862332105637,
+      "kl": 0.024383544921875,
+      "learning_rate": 2.5534962781354317e-06,
+      "loss": 0.0087,
+      "num_tokens": 3088262.0,
+      "reward": 0.26893792301416397,
+      "reward_std": 0.38880206644535065,
+      "rewards/code_reward": 0.19893791526556015,
+      "rewards/format_reward": 0.7000000178813934,
+      "step": 271
+    },
+    {
+      "clip_ratio": 0.0009908578940667212,
+      "epoch": 0.005074390187024859,
+      "grad_norm": 0.09044025838375092,
+      "kl": 0.0234375,
+      "learning_rate": 2.538981791986634e-06,
+      "loss": 0.0088,
+      "step": 272
+    },
+    {
+      "clip_ratio": 0.0006488285725936294,
+      "epoch": 0.0050930460333006855,
+      "grad_norm": 0.0915515273809433,
+      "kl": 0.022735595703125,
+      "learning_rate": 2.524476159739218e-06,
+      "loss": 0.0086,
+      "step": 273
+    },
+    {
+      "clip_ratio": 0.00046395487152040005,
+      "completion_length": 707.9583740234375,
+      "epoch": 0.005111701879576512,
+      "grad_norm": 0.08589129149913788,
+      "kl": 0.02691650390625,
+      "learning_rate": 2.5099799900204607e-06,
+      "loss": -0.013,
+      "num_tokens": 3119713.0,
+      "reward": 0.7045832872390747,
+      "reward_std": 0.39258189499378204,
+      "rewards/code_reward": 0.6250000149011612,
+      "rewards/format_reward": 0.7958333194255829,
+      "step": 274
+    },
+    {
+      "clip_ratio": 0.0006349159230012447,
+      "epoch": 0.005130357725852339,
+      "grad_norm": 0.08367536962032318,
+      "kl": 0.02716064453125,
+      "learning_rate": 2.4954938910606108e-06,
+      "loss": -0.013,
+      "step": 275
+    },
+    {
+      "clip_ratio": 0.0007932094158604741,
+      "epoch": 0.005149013572128166,
+      "grad_norm": 0.08294366300106049,
+      "kl": 0.024322509765625,
+      "learning_rate": 2.481018470667368e-06,
+      "loss": -0.0131,
+      "step": 276
+    },
+    {
+      "clip_ratio": 0.0012548710801638663,
+      "completion_length": 581.9166870117188,
+      "epoch": 0.005167669418403993,
+      "grad_norm": 0.09755374491214752,
+      "kl": 0.02392578125,
+      "learning_rate": 2.4665543362003802e-06,
+      "loss": 0.0183,
+      "num_tokens": 3144827.0,
+      "reward": 0.7179166376590729,
+      "reward_std": 0.27089906856417656,
+      "rewards/code_reward": 0.6666666716337204,
+      "rewards/format_reward": 0.5124999731779099,
+      "step": 277
+    },
+    {
+      "clip_ratio": 0.001595219480805099,
+      "epoch": 0.005186325264679819,
+      "grad_norm": 0.09835170209407806,
+      "kl": 0.0220947265625,
+      "learning_rate": 2.4521020945457615e-06,
+      "loss": 0.0181,
+      "step": 278
+    },
+    {
+      "clip_ratio": 0.0019812395912595093,
+      "epoch": 0.005204981110955646,
+      "grad_norm": 0.0999624952673912,
+      "kl": 0.0208740234375,
+      "learning_rate": 2.4376623520906255e-06,
+      "loss": 0.0182,
+      "step": 279
+    },
+    {
+      "clip_ratio": 0.0011003862309735268,
+      "completion_length": 1171.5833435058594,
+      "epoch": 0.005223636957231472,
+      "grad_norm": 0.07472976297140121,
+      "kl": 0.009613037109375,
+      "learning_rate": 2.4232357146976478e-06,
+      "loss": 0.0011,
+      "num_tokens": 3189013.0,
+      "reward": 0.5404510945081711,
+      "reward_std": 0.15828495100140572,
+      "rewards/code_reward": 0.4721178039908409,
+      "rewards/format_reward": 0.6833333224058151,
+      "step": 280
+    },
+    {
+      "clip_ratio": 0.0007794810517225415,
+      "epoch": 0.005242292803507299,
+      "grad_norm": 0.07738126814365387,
+      "kl": 0.009857177734375,
+      "learning_rate": 2.408822787679637e-06,
+      "loss": 0.0013,
+      "step": 281
+    },
+    {
+      "clip_ratio": 0.0008492782071698457,
+      "epoch": 0.005260948649783125,
+      "grad_norm": 0.07533033192157745,
+      "kl": 0.009613037109375,
+      "learning_rate": 2.3944241757741475e-06,
+      "loss": 0.001,
+      "step": 282
+    },
+    {
+      "clip_ratio": 0.0003291434477432631,
+      "completion_length": 672.625,
+      "epoch": 0.005279604496058953,
+      "grad_norm": 0.07457376271486282,
+      "kl": 0.012908935546875,
+      "learning_rate": 2.380040483118097e-06,
+      "loss": -0.0139,
+      "num_tokens": 3212848.0,
+      "reward": 1.0279166400432587,
+      "reward_std": 0.17488877102732658,
+      "rewards/code_reward": 0.9583333432674408,
+      "rewards/format_reward": 0.6958333253860474,
+      "step": 283
+    },
+    {
+      "clip_ratio": 5.457324004964903e-05,
+      "epoch": 0.005298260342334779,
+      "grad_norm": 0.07336500287055969,
+      "kl": 0.0133056640625,
+      "learning_rate": 2.365672313222419e-06,
+      "loss": -0.0141,
+      "step": 284
+    },
+    {
+      "clip_ratio": 0.0002531533937144559,
+      "epoch": 0.005316916188610606,
+      "grad_norm": 0.07265036553144455,
+      "kl": 0.0135498046875,
+      "learning_rate": 2.351320268946749e-06,
+      "loss": -0.014,
+      "step": 285
+    },
+    {
+      "clip_ratio": 0.0013167554279789329,
+      "completion_length": 909.7083740234375,
+      "epoch": 0.0053355720348864325,
+      "grad_norm": 0.11532127857208252,
+      "kl": 0.01422119140625,
+      "learning_rate": 2.336984952474119e-06,
+      "loss": -0.0194,
+      "num_tokens": 3247833.0,
+      "reward": 0.09151961281895638,
+      "reward_std": 0.09498994797468185,
+      "rewards/code_reward": 0.04901960864663124,
+      "rewards/format_reward": 0.42499999701976776,
+      "step": 286
+    },
+    {
+      "clip_ratio": 0.0011718517343979329,
+      "epoch": 0.005354227881162259,
+      "grad_norm": 0.10550913959741592,
+      "kl": 0.013946533203125,
+      "learning_rate": 2.322666965285697e-06,
+      "loss": -0.0197,
+      "step": 287
+    },
+    {
+      "clip_ratio": 0.0012868313351646066,
+      "epoch": 0.005372883727438086,
+      "grad_norm": 0.10481560975313187,
+      "kl": 0.015716552734375,
+      "learning_rate": 2.3083669081355507e-06,
+      "loss": -0.0198,
+      "step": 288
+    },
+    {
+      "clip_ratio": 0.00034923299972433597,
+      "completion_length": 1332.0833740234375,
+      "epoch": 0.005391539573713912,
+      "grad_norm": 0.06407570093870163,
+      "kl": 0.0085906982421875,
+      "learning_rate": 2.2940853810254377e-06,
+      "loss": 0.0465,
+      "num_tokens": 3299579.0,
+      "reward": 0.05910714715719223,
+      "reward_std": 0.04962867684662342,
+      "rewards/code_reward": 0.0053571430034935474,
+      "rewards/format_reward": 0.5374999940395355,
+      "step": 289
+    },
+    {
+      "clip_ratio": 0.0005735456070397049,
+      "epoch": 0.00541019541998974,
+      "grad_norm": 0.08031246811151505,
+      "kl": 0.00830078125,
+      "learning_rate": 2.2798229831796313e-06,
+      "loss": 0.0464,
+      "step": 290
+    },
+    {
+      "clip_ratio": 0.0007879029144532979,
+      "epoch": 0.005428851266265566,
+      "grad_norm": 0.0649939477443695,
+      "kl": 0.0087432861328125,
+      "learning_rate": 2.2655803130197816e-06,
+      "loss": 0.0464,
+      "step": 291
+    },
+    {
+      "clip_ratio": 0.00107748550362885,
+      "completion_length": 1028.0833740234375,
+      "epoch": 0.005447507112541393,
+      "grad_norm": 0.10151123255491257,
+      "kl": 0.0071563720703125,
+      "learning_rate": 2.2513579681398034e-06,
+      "loss": 0.0503,
+      "num_tokens": 3348829.0,
+      "reward": 0.1927850916981697,
+      "reward_std": 0.3257390707731247,
+      "rewards/code_reward": 0.13903508707880974,
+      "rewards/format_reward": 0.5375000089406967,
+      "step": 292
+    },
+    {
+      "clip_ratio": 0.0008358334598597139,
+      "epoch": 0.005466162958817219,
+      "grad_norm": 0.09055967628955841,
+      "kl": 0.0071868896484375,
+      "learning_rate": 2.237156545280803e-06,
+      "loss": 0.0504,
+      "step": 293
+    },
+    {
+      "clip_ratio": 0.0009988098172470927,
+      "epoch": 0.005484818805093046,
+      "grad_norm": 0.08410565555095673,
+      "kl": 0.0072174072265625,
+      "learning_rate": 2.2229766403060403e-06,
+      "loss": 0.0501,
+      "step": 294
+    },
+    {
+      "clip_ratio": 0.001458263781387359,
+      "completion_length": 681.2083740234375,
+      "epoch": 0.005503474651368872,
+      "grad_norm": 0.09818808734416962,
+      "kl": 0.01514434814453125,
+      "learning_rate": 2.2088188481759305e-06,
+      "loss": 0.0187,
+      "num_tokens": 3378258.0,
+      "reward": 0.5579166747629642,
+      "reward_std": 0.045613983646035194,
+      "rewards/code_reward": 0.5,
+      "rewards/format_reward": 0.5791666805744171,
+      "step": 295
+    },
+    {
+      "clip_ratio": 0.0013955137401353568,
+      "epoch": 0.005522130497644699,
+      "grad_norm": 0.09592823684215546,
+      "kl": 0.017864227294921875,
+      "learning_rate": 2.194683762923073e-06,
+      "loss": 0.0188,
+      "step": 296
+    },
+    {
+      "clip_ratio": 0.0017046461580321193,
+      "epoch": 0.005540786343920526,
+      "grad_norm": 0.09802346676588058,
+      "kl": 0.01699066162109375,
+      "learning_rate": 2.1805719776273387e-06,
+      "loss": 0.0187,
+      "step": 297
+    },
+    {
+      "clip_ratio": 0.0009031847293954343,
+      "completion_length": 919.2917175292969,
+      "epoch": 0.005559442190196353,
+      "grad_norm": 0.08558425307273865,
+      "kl": 0.012786865234375,
+      "learning_rate": 2.166484084390974e-06,
+      "loss": -0.0303,
+      "num_tokens": 3419881.0,
+      "reward": 0.5293938517570496,
+      "reward_std": 0.3867318332195282,
+      "rewards/code_reward": 0.4823105186223984,
+      "rewards/format_reward": 0.47083334624767303,
+      "step": 298
+    },
+    {
+      "clip_ratio": 0.0009096206922549754,
+      "epoch": 0.0055780980364721795,
+      "grad_norm": 0.0866839662194252,
+      "kl": 0.0133056640625,
+      "learning_rate": 2.1524206743137636e-06,
+      "loss": -0.0303,
+      "step": 299
+    },
+    {
+      "clip_ratio": 0.00072934762283694,
+      "epoch": 0.005596753882748006,
+      "grad_norm": 0.08566750586032867,
+      "kl": 0.0135498046875,
+      "learning_rate": 2.1383823374682287e-06,
+      "loss": -0.0304,
+      "step": 300
+    },
+    {
+      "clip_ratio": 0.0010719166893977672,
+      "completion_length": 849.1666870117188,
+      "epoch": 0.005615409729023833,
+      "grad_norm": 0.09118139743804932,
+      "kl": 0.0116424560546875,
+      "learning_rate": 2.124369662874868e-06,
+      "loss": 0.0616,
+      "num_tokens": 3455405.0,
+      "reward": 0.5117070078849792,
+      "reward_std": 0.4537804424762726,
+      "rewards/code_reward": 0.43629033863544464,
+      "rewards/format_reward": 0.7541666626930237,
+      "step": 301
+    },
+    {
+      "clip_ratio": 0.0009527777438051999,
+      "epoch": 0.005634065575299659,
+      "grad_norm": 0.09360919892787933,
+      "kl": 0.0117034912109375,
+      "learning_rate": 2.110383238477441e-06,
+      "loss": 0.0618,
+      "step": 302
+    },
+    {
+      "clip_ratio": 0.0010622217669151723,
+      "epoch": 0.005652721421575487,
+      "grad_norm": 0.09148411452770233,
+      "kl": 0.0118560791015625,
+      "learning_rate": 2.096423651118305e-06,
+      "loss": 0.0616,
+      "step": 303
+    },
+    {
+      "clip_ratio": 0.0005411781894508749,
+      "completion_length": 1098.7083740234375,
+      "epoch": 0.005671377267851313,
+      "grad_norm": 0.08525364845991135,
+      "kl": 0.014556884765625,
+      "learning_rate": 2.082491486513788e-06,
+      "loss": 0.0016,
+      "num_tokens": 3502546.0,
+      "reward": 0.3728175610303879,
+      "reward_std": 0.3381016403436661,
+      "rewards/code_reward": 0.321984238922596,
+      "rewards/format_reward": 0.5083333253860474,
+      "step": 304
+    },
+    {
+      "clip_ratio": 0.0007919937779661268,
+      "epoch": 0.00569003311412714,
+      "grad_norm": 0.40698713064193726,
+      "kl": 0.01470947265625,
+      "learning_rate": 2.0685873292296116e-06,
+      "loss": 0.0018,
+      "step": 305
+    },
+    {
+      "clip_ratio": 0.0005383995739975944,
+      "epoch": 0.005708688960402966,
+      "grad_norm": 0.49188467860221863,
+      "kl": 0.014678955078125,
+      "learning_rate": 2.054711762656369e-06,
+      "loss": 0.0018,
+      "step": 306
+    },
+    {
+      "clip_ratio": 0.0010625849245116115,
+      "completion_length": 683.9583435058594,
+      "epoch": 0.005727344806678793,
+      "grad_norm": 0.10668221861124039,
+      "kl": 0.014434814453125,
+      "learning_rate": 2.040865368985044e-06,
+      "loss": -0.0085,
+      "num_tokens": 3530901.0,
+      "reward": 0.776462584733963,
+      "reward_std": 0.17944780550897121,
+      "rewards/code_reward": 0.7243793159723282,
+      "rewards/format_reward": 0.5208333432674408,
+      "step": 307
+    },
+    {
+      "clip_ratio": 0.0016164338449016213,
+      "epoch": 0.005746000652954619,
+      "grad_norm": 0.10843304544687271,
+      "kl": 0.0142822265625,
+      "learning_rate": 2.027048729182583e-06,
+      "loss": -0.0086,
+      "step": 308
+    },
+    {
+      "clip_ratio": 0.001167797192465514,
+      "epoch": 0.005764656499230446,
+      "grad_norm": 0.10622025281190872,
+      "kl": 0.014434814453125,
+      "learning_rate": 2.0132624229675205e-06,
+      "loss": -0.0089,
+      "step": 309
+    },
+    {
+      "clip_ratio": 0.0011982419237028807,
+      "completion_length": 876.9166870117188,
+      "epoch": 0.005783312345506273,
+      "grad_norm": 0.09164712578058243,
+      "kl": 0.021636962890625,
+      "learning_rate": 1.9995070287856546e-06,
+      "loss": -0.0053,
+      "num_tokens": 3564103.0,
+      "reward": 0.5786574631929398,
+      "reward_std": 0.41006001830101013,
+      "rewards/code_reward": 0.5207407623529434,
+      "rewards/format_reward": 0.5791666507720947,
+      "step": 310
+    },
+    {
+      "clip_ratio": 0.001371737860608846,
+      "epoch": 0.0058019681917821,
+      "grad_norm": 0.09242158383131027,
+      "kl": 0.022064208984375,
+      "learning_rate": 1.985783123785774e-06,
+      "loss": -0.0055,
+      "step": 311
+    },
+    {
+      "clip_ratio": 0.001422962493961677,
+      "epoch": 0.0058206240380579265,
+      "grad_norm": 0.09199241548776627,
+      "kl": 0.021728515625,
+      "learning_rate": 1.9720912837954486e-06,
+      "loss": -0.0054,
+      "step": 312
+    },
+    {
+      "clip_ratio": 0.0006902147724758834,
+      "completion_length": 927.7500305175781,
+      "epoch": 0.005839279884333753,
+      "grad_norm": 0.09657740592956543,
+      "kl": 0.0142822265625,
+      "learning_rate": 1.958432083296862e-06,
+      "loss": -0.0334,
+      "num_tokens": 3595285.0,
+      "reward": 0.4076350927352905,
+      "reward_std": 0.3373851850628853,
+      "rewards/code_reward": 0.3297183997929096,
+      "rewards/format_reward": 0.7791666686534882,
+      "step": 313
+    },
+    {
+      "clip_ratio": 0.0006798989197704941,
+      "epoch": 0.00585793573060958,
+      "grad_norm": 0.09554401785135269,
+      "kl": 0.013671875,
+      "learning_rate": 1.9448060954027093e-06,
+      "loss": -0.0335,
+      "step": 314
+    },
+    {
+      "clip_ratio": 0.0008473133784718812,
+      "epoch": 0.005876591576885406,
+      "grad_norm": 0.09273131936788559,
+      "kl": 0.014678955078125,
+      "learning_rate": 1.931213891832153e-06,
+      "loss": -0.0337,
+      "step": 315
+    },
+    {
+      "clip_ratio": 0.0005323913064785302,
+      "completion_length": 834.8333740234375,
+      "epoch": 0.005895247423161233,
+      "grad_norm": 0.10354404151439667,
+      "kl": 0.025360107421875,
+      "learning_rate": 1.9176560428868336e-06,
+      "loss": -0.0085,
+      "num_tokens": 3639501.0,
+      "reward": 0.64698725938797,
+      "reward_std": 0.4041699767112732,
+      "rewards/code_reward": 0.5749039202928543,
+      "rewards/format_reward": 0.7208333015441895,
+      "step": 316
+    },
+    {
+      "clip_ratio": 0.0008116684039123356,
+      "epoch": 0.00591390326943706,
+      "grad_norm": 0.10160063952207565,
+      "kl": 0.025299072265625,
+      "learning_rate": 1.9041331174269373e-06,
+      "loss": -0.0088,
+      "step": 317
+    },
+    {
+      "clip_ratio": 0.0005085803277324885,
+      "epoch": 0.005932559115712887,
+      "grad_norm": 0.09548759460449219,
+      "kl": 0.0244140625,
+      "learning_rate": 1.8906456828473341e-06,
+      "loss": -0.009,
+      "step": 318
+    },
+    {
+      "clip_ratio": 0.0008387799316551536,
+      "completion_length": 1620.8750915527344,
+      "epoch": 0.005951214961988713,
+      "grad_norm": 0.07884962111711502,
+      "kl": 0.0074615478515625,
+      "learning_rate": 1.8771943050537656e-06,
+      "loss": 0.0221,
+      "num_tokens": 3698802.0,
+      "reward": 0.38234737887978554,
+      "reward_std": 0.20284880511462688,
+      "rewards/code_reward": 0.30693069100379944,
+      "rewards/format_reward": 0.7541666626930237,
+      "step": 319
+    },
+    {
+      "clip_ratio": 0.0007986939453985542,
+      "epoch": 0.00596987080826454,
+      "grad_norm": 0.07618702203035355,
+      "kl": 0.007415771484375,
+      "learning_rate": 1.8637795484391046e-06,
+      "loss": 0.0221,
+      "step": 320
+    },
+    {
+      "clip_ratio": 0.0007583920087199658,
+      "epoch": 0.0059885266545403664,
+      "grad_norm": 0.0829911157488823,
+      "kl": 0.0075225830078125,
+      "learning_rate": 1.8504019758596698e-06,
+      "loss": 0.0221,
+      "step": 321
+    },
+    {
+      "clip_ratio": 0.001169055758509785,
+      "completion_length": 795.7916717529297,
+      "epoch": 0.006007182500816193,
+      "grad_norm": 0.14611609280109406,
+      "kl": 0.019012451171875,
+      "learning_rate": 1.8370621486116163e-06,
+      "loss": 0.023,
+      "num_tokens": 3734797.0,
+      "reward": 0.645012378692627,
+      "reward_std": 0.2191050536930561,
+      "rewards/code_reward": 0.5820957124233246,
+      "rewards/format_reward": 0.6291666924953461,
+      "step": 322
+    },
+    {
+      "clip_ratio": 0.0009497803403064609,
+      "epoch": 0.0060258383470920196,
+      "grad_norm": 0.10970307886600494,
+      "kl": 0.02032470703125,
+      "learning_rate": 1.823760626407377e-06,
+      "loss": 0.0227,
+      "step": 323
+    },
+    {
+      "clip_ratio": 0.0009026691550388932,
+      "epoch": 0.006044494193367847,
+      "grad_norm": 0.11829708516597748,
+      "kl": 0.0189208984375,
+      "learning_rate": 1.8104979673521838e-06,
+      "loss": 0.023,
+      "step": 324
+    },
+    {
+      "clip_ratio": 0.0008327370742335916,
+      "completion_length": 874.0416870117188,
+      "epoch": 0.0060631500396436736,
+      "grad_norm": 0.09914888441562653,
+      "kl": 0.02264404296875,
+      "learning_rate": 1.7972747279206482e-06,
+      "loss": -0.0067,
+      "num_tokens": 3769226.0,
+      "reward": 0.4188089519739151,
+      "reward_std": 0.19916179403662682,
+      "rewards/code_reward": 0.36964227352291346,
+      "rewards/format_reward": 0.49166665971279144,
+      "step": 325
+    },
+    {
+      "clip_ratio": 0.0008456225623376667,
+      "epoch": 0.0060818058859195,
+      "grad_norm": 0.09671460092067719,
+      "kl": 0.02288818359375,
+      "learning_rate": 1.7840914629334122e-06,
+      "loss": -0.0064,
+      "step": 326
+    },
+    {
+      "clip_ratio": 0.0010657160892151296,
+      "epoch": 0.006100461732195327,
+      "grad_norm": 0.09105102717876434,
+      "kl": 0.02294921875,
+      "learning_rate": 1.7709487255338731e-06,
+      "loss": -0.0066,
+      "step": 327
+    },
+    {
+      "clip_ratio": 0.0010840718168765306,
+      "completion_length": 422.5833435058594,
+      "epoch": 0.006119117578471153,
+      "grad_norm": 0.13927879929542542,
+      "kl": 0.019073486328125,
+      "learning_rate": 1.7578470671649684e-06,
+      "loss": 0.0094,
+      "num_tokens": 3789448.0,
+      "reward": 0.5294444337487221,
+      "reward_std": 0.17376149259507656,
+      "rewards/code_reward": 0.4444444179534912,
+      "rewards/format_reward": 0.8499999940395355,
+      "step": 328
+    },
+    {
+      "clip_ratio": 0.000780029222369194,
+      "epoch": 0.00613777342474698,
+      "grad_norm": 0.13586357235908508,
+      "kl": 0.021209716796875,
+      "learning_rate": 1.744787037546045e-06,
+      "loss": 0.0092,
+      "step": 329
+    },
+    {
+      "clip_ratio": 0.001488116628024727,
+      "epoch": 0.006156429271022806,
+      "grad_norm": 0.16369539499282837,
+      "kl": 0.02215576171875,
+      "learning_rate": 1.731769184649788e-06,
+      "loss": 0.0091,
+      "step": 330
+    },
+    {
+      "clip_ratio": 0.0005380379006965086,
+      "completion_length": 776.1250305175781,
+      "epoch": 0.006175085117298634,
+      "grad_norm": 0.0828891322016716,
+      "kl": 0.02215576171875,
+      "learning_rate": 1.7187940546792325e-06,
+      "loss": -0.0154,
+      "num_tokens": 3825619.0,
+      "reward": 0.6713247746229172,
+      "reward_std": 0.09931758604943752,
+      "rewards/code_reward": 0.5779914557933807,
+      "rewards/format_reward": 0.9333333075046539,
+      "step": 331
+    },
+    {
+      "clip_ratio": 0.0008244940836448222,
+      "epoch": 0.00619374096357446,
+      "grad_norm": 0.08281542360782623,
+      "kl": 0.02386474609375,
+      "learning_rate": 1.7058621920448465e-06,
+      "loss": -0.0155,
+      "step": 332
+    },
+    {
+      "clip_ratio": 0.0006863593007437885,
+      "epoch": 0.006212396809850287,
+      "grad_norm": 0.08620591461658478,
+      "kl": 0.027679443359375,
+      "learning_rate": 1.6929741393416855e-06,
+      "loss": -0.0155,
+      "step": 333
+    },
+    {
+      "clip_ratio": 0.0011899169767275453,
+      "completion_length": 866.6250305175781,
+      "epoch": 0.0062310526561261135,
+      "grad_norm": 0.09219539165496826,
+      "kl": 0.034088134765625,
+      "learning_rate": 1.6801304373266286e-06,
+      "loss": 0.0009,
+      "num_tokens": 3860470.0,
+      "reward": 0.526757076382637,
+      "reward_std": 0.33820731937885284,
+      "rewards/code_reward": 0.45467372983694077,
+      "rewards/format_reward": 0.7208333313465118,
+      "step": 334
+    },
+    {
+      "clip_ratio": 0.0009668752900324762,
+      "epoch": 0.00624970850240194,
+      "grad_norm": 0.09143849462270737,
+      "kl": 0.030029296875,
+      "learning_rate": 1.667331624895689e-06,
+      "loss": 0.0004,
+      "step": 335
+    },
+    {
+      "clip_ratio": 0.0009758240194059908,
+      "epoch": 0.006268364348677767,
+      "grad_norm": 0.09234081953763962,
+      "kl": 0.03564453125,
+      "learning_rate": 1.6545782390614037e-06,
+      "loss": 0.0005,
+      "step": 336
+    },
+    {
+      "clip_ratio": 0.0009786576265469193,
+      "completion_length": 767.2083740234375,
+      "epoch": 0.006287020194953594,
+      "grad_norm": 0.11128581315279007,
+      "kl": 0.0589599609375,
+      "learning_rate": 1.6418708149302992e-06,
+      "loss": 0.0293,
+      "num_tokens": 3900099.0,
+      "reward": 0.5688367038965225,
+      "reward_std": 0.1789155676960945,
+      "rewards/code_reward": 0.49841997027397156,
+      "rewards/format_reward": 0.7041666805744171,
+      "step": 337
+    },
+    {
+      "clip_ratio": 0.0011234881239943206,
+      "epoch": 0.0063056760412294206,
+      "grad_norm": 0.11240074783563614,
+      "kl": 0.0528564453125,
+      "learning_rate": 1.6292098856804423e-06,
+      "loss": 0.029,
+      "step": 338
+    },
+    {
+      "clip_ratio": 0.0006711997266393155,
+      "epoch": 0.006324331887505247,
+      "grad_norm": 0.10572442412376404,
+      "kl": 0.05133056640625,
+      "learning_rate": 1.6165959825390661e-06,
+      "loss": 0.0289,
+      "step": 339
+    },
+    {
+      "clip_ratio": 0.0007651688065379858,
+      "completion_length": 662.3333587646484,
+      "epoch": 0.006342987733781074,
+      "grad_norm": 0.09474088996648788,
+      "kl": 0.0198974609375,
+      "learning_rate": 1.604029634760284e-06,
+      "loss": 0.0135,
+      "num_tokens": 3926987.0,
+      "reward": 0.8191105425357819,
+      "reward_std": 0.3644227534532547,
+      "rewards/code_reward": 0.7049438655376434,
+      "rewards/format_reward": 1.1416666507720947,
+      "step": 340
+    },
+    {
+      "clip_ratio": 0.0006743039994034916,
+      "epoch": 0.0063616435800569,
+      "grad_norm": 0.09659333527088165,
+      "kl": 0.020263671875,
+      "learning_rate": 1.59151136960288e-06,
+      "loss": 0.0139,
+      "step": 341
+    },
+    {
+      "clip_ratio": 0.0006922298634890467,
+      "epoch": 0.006380299426332727,
+      "grad_norm": 0.09575100243091583,
+      "kl": 0.01959228515625,
+      "learning_rate": 1.5790417123081903e-06,
+      "loss": 0.0135,
+      "step": 342
+    },
+    {
+      "clip_ratio": 0.0005754130106652156,
+      "completion_length": 964.3333740234375,
+      "epoch": 0.006398955272608553,
+      "grad_norm": 0.08601634949445724,
+      "kl": 0.012359619140625,
+      "learning_rate": 1.5666211860780583e-06,
+      "loss": -0.0431,
+      "num_tokens": 3969595.0,
+      "reward": 0.43568913638591766,
+      "reward_std": 0.2704745829105377,
+      "rewards/code_reward": 0.3594391196966171,
+      "rewards/format_reward": 0.762499988079071,
+      "step": 343
+    },
+    {
+      "clip_ratio": 0.0005838508659508079,
+      "epoch": 0.006417611118884381,
+      "grad_norm": 0.08448081463575363,
+      "kl": 0.01171875,
+      "learning_rate": 1.5542503120528918e-06,
+      "loss": -0.043,
+      "step": 344
+    },
+    {
+      "clip_ratio": 0.0007864527578931302,
+      "epoch": 0.006436266965160207,
+      "grad_norm": 0.08665861189365387,
+      "kl": 0.011932373046875,
+      "learning_rate": 1.5419296092897866e-06,
+      "loss": -0.0432,
+      "step": 345
+    },
+    {
+      "clip_ratio": 0.0012224867241457105,
+      "completion_length": 830.8750305175781,
+      "epoch": 0.006454922811436034,
+      "grad_norm": 0.08804712444543839,
+      "kl": 0.03411865234375,
+      "learning_rate": 1.529659594740755e-06,
+      "loss": 0.0054,
+      "num_tokens": 3999184.0,
+      "reward": 0.12666667252779007,
+      "reward_std": 0.1735054012387991,
+      "rewards/code_reward": 0.0416666679084301,
+      "rewards/format_reward": 0.8499999642372131,
+      "step": 346
+    },
+    {
+      "clip_ratio": 0.0012100348249077797,
+      "epoch": 0.0064735786577118605,
+      "grad_norm": 0.08527618646621704,
+      "kl": 0.03369140625,
+      "learning_rate": 1.5174407832310338e-06,
+      "loss": 0.0053,
+      "step": 347
+    },
+    {
+      "clip_ratio": 0.0010465410014148802,
+      "epoch": 0.006492234503987687,
+      "grad_norm": 0.08590537309646606,
+      "kl": 0.03118896484375,
+      "learning_rate": 1.5052736874374815e-06,
+      "loss": 0.0055,
+      "step": 348
+    },
+    {
+      "clip_ratio": 0.001526809181086719,
+      "completion_length": 865.7500610351562,
+      "epoch": 0.006510890350263514,
+      "grad_norm": 0.09376516193151474,
+      "kl": 0.0206298828125,
+      "learning_rate": 1.4931588178670695e-06,
+      "loss": 0.0089,
+      "num_tokens": 4031674.0,
+      "reward": 0.2808653935790062,
+      "reward_std": 0.27811707369983196,
+      "rewards/code_reward": 0.21794871985912323,
+      "rewards/format_reward": 0.6291666775941849,
+      "step": 349
+    },
+    {
+      "clip_ratio": 0.000961648824159056,
+      "epoch": 0.00652954619653934,
+      "grad_norm": 0.10522089153528214,
+      "kl": 0.02142333984375,
+      "learning_rate": 1.4810966828354605e-06,
+      "loss": 0.0089,
+      "step": 350
+    },
+    {
+      "clip_ratio": 0.0010438500612508506,
+      "epoch": 0.006548202042815168,
+      "grad_norm": 0.09141746908426285,
+      "kl": 0.0203857421875,
+      "learning_rate": 1.469087788445684e-06,
+      "loss": 0.0084,
+      "step": 351
+    },
+    {
+      "clip_ratio": 0.0010631512850522995,
+      "completion_length": 1097.8333740234375,
+      "epoch": 0.006566857889090994,
+      "grad_norm": 0.08537644892930984,
+      "kl": 0.014739990234375,
+      "learning_rate": 1.4571326385668965e-06,
+      "loss": -0.0225,
+      "num_tokens": 4071702.0,
+      "reward": 0.3495173156261444,
+      "reward_std": 0.350162148475647,
+      "rewards/code_reward": 0.312017310410738,
+      "rewards/format_reward": 0.3749999925494194,
+      "step": 352
+    },
+    {
+      "clip_ratio": 0.0007791414973326027,
+      "epoch": 0.006585513735366821,
+      "grad_norm": 0.08616505563259125,
+      "kl": 0.015045166015625,
+      "learning_rate": 1.4452317348132434e-06,
+      "loss": -0.0226,
+      "step": 353
+    },
+    {
+      "clip_ratio": 0.0008178194111678749,
+      "epoch": 0.006604169581642647,
+      "grad_norm": 0.08682885020971298,
+      "kl": 0.014923095703125,
+      "learning_rate": 1.4333855765228104e-06,
+      "loss": -0.0226,
+      "step": 354
+    },
+    {
+      "clip_ratio": 0.0010338413121644408,
+      "completion_length": 677.5,
+      "epoch": 0.006622825427918474,
+      "grad_norm": 0.08901394158601761,
+      "kl": 0.0438232421875,
+      "learning_rate": 1.421594660736675e-06,
+      "loss": 0.0347,
+      "num_tokens": 4095762.0,
+      "reward": 0.6164166405797005,
+      "reward_std": 0.12990031018853188,
+      "rewards/code_reward": 0.539749950170517,
+      "rewards/format_reward": 0.7666666805744171,
+      "step": 355
+    },
+    {
+      "clip_ratio": 0.0012019489658996463,
+      "epoch": 0.0066414812741943,
+      "grad_norm": 0.08767423033714294,
+      "kl": 0.03948974609375,
+      "learning_rate": 1.4098594821780476e-06,
+      "loss": 0.0344,
+      "step": 356
+    },
+    {
+      "clip_ratio": 0.001214035670273006,
+      "epoch": 0.006660137120470127,
+      "grad_norm": 0.09001511335372925,
+      "kl": 0.038970947265625,
+      "learning_rate": 1.3981805332315174e-06,
+      "loss": 0.0346,
+      "step": 357
+    },
+    {
+      "clip_ratio": 0.0010124409454874694,
+      "completion_length": 1344.5417175292969,
+      "epoch": 0.006678792966745954,
+      "grad_norm": 0.07684619724750519,
+      "kl": 0.02001190185546875,
+      "learning_rate": 1.3865583039223929e-06,
+      "loss": 0.0339,
+      "num_tokens": 4142671.0,
+      "reward": 0.7462417930364609,
+      "reward_std": 0.22497505508363247,
+      "rewards/code_reward": 0.6674917489290237,
+      "rewards/format_reward": 0.7874999940395355,
+      "step": 358
+    },
+    {
+      "clip_ratio": 0.0007824611384421587,
+      "epoch": 0.006697448813021781,
+      "grad_norm": 0.0674465075135231,
+      "kl": 0.0182037353515625,
+      "learning_rate": 1.374993281896137e-06,
+      "loss": 0.0337,
+      "step": 359
+    },
+    {
+      "clip_ratio": 0.0010956324986182153,
+      "epoch": 0.0067161046592976075,
+      "grad_norm": 0.0694521889090538,
+      "kl": 0.01786041259765625,
+      "learning_rate": 1.3634859523979134e-06,
+      "loss": 0.034,
+      "step": 360
+    },
+    {
+      "clip_ratio": 0.0011445771087892354,
+      "completion_length": 637.5833740234375,
+      "epoch": 0.006734760505573434,
+      "grad_norm": 0.11650668829679489,
+      "kl": 0.012542724609375,
+      "learning_rate": 1.3520367982522208e-06,
+      "loss": -0.0048,
+      "num_tokens": 4168125.0,
+      "reward": 0.3813568465411663,
+      "reward_std": 0.24762929044663906,
+      "rewards/code_reward": 0.29594019055366516,
+      "rewards/format_reward": 0.8541666567325592,
+      "step": 361
+    },
+    {
+      "clip_ratio": 0.0014873360050842166,
+      "epoch": 0.006753416351849261,
+      "grad_norm": 0.11154331266880035,
+      "kl": 0.01220703125,
+      "learning_rate": 1.3406462998426358e-06,
+      "loss": -0.0047,
+      "step": 362
+    },
+    {
+      "clip_ratio": 0.0012732939212583005,
+      "epoch": 0.006772072198125087,
+      "grad_norm": 0.11064095795154572,
+      "kl": 0.01312255859375,
+      "learning_rate": 1.3293149350916595e-06,
+      "loss": -0.0045,
+      "step": 363
+    },
+    {
+      "clip_ratio": 0.0010022299247793853,
+      "completion_length": 581.25,
+      "epoch": 0.006790728044400914,
+      "grad_norm": 0.20353944599628448,
+      "kl": 0.017822265625,
+      "learning_rate": 1.3180431794406623e-06,
+      "loss": 0.0111,
+      "num_tokens": 4198479.0,
+      "reward": 0.10175329819321632,
+      "reward_std": 0.05031761899590492,
+      "rewards/code_reward": 0.033003296703100204,
+      "rewards/format_reward": 0.6875,
+      "step": 364
+    },
+    {
+      "clip_ratio": 0.0008599580323789269,
+      "epoch": 0.006809383890676741,
+      "grad_norm": 0.1254553347826004,
+      "kl": 0.0179443359375,
+      "learning_rate": 1.3068315058299358e-06,
+      "loss": 0.0109,
+      "step": 365
+    },
+    {
+      "clip_ratio": 0.0007180549946497194,
+      "epoch": 0.006828039736952568,
+      "grad_norm": 0.11537165939807892,
+      "kl": 0.01995849609375,
+      "learning_rate": 1.2956803846788503e-06,
+      "loss": 0.011,
+      "step": 366
+    },
+    {
+      "clip_ratio": 0.0006652901647612453,
+      "completion_length": 934.2917175292969,
+      "epoch": 0.006846695583228394,
+      "grad_norm": 0.09258420765399933,
+      "kl": 0.011566162109375,
+      "learning_rate": 1.284590283866116e-06,
+      "loss": 0.0124,
+      "num_tokens": 4241782.0,
+      "reward": 0.21692460775375366,
+      "reward_std": 0.24195446819067,
+      "rewards/code_reward": 0.14484127657487988,
+      "rewards/format_reward": 0.7208333313465118,
+      "step": 367
+    },
+    {
+      "clip_ratio": 0.0009541537729091942,
+      "epoch": 0.006865351429504221,
+      "grad_norm": 0.09379203617572784,
+      "kl": 0.011322021484375,
+      "learning_rate": 1.2735616687101518e-06,
+      "loss": 0.0125,
+      "step": 368
+    },
+    {
+      "clip_ratio": 0.0010697944671846926,
+      "epoch": 0.006884007275780047,
+      "grad_norm": 0.09176654368638992,
+      "kl": 0.01129150390625,
+      "learning_rate": 1.2625950019495614e-06,
+      "loss": 0.0124,
+      "step": 369
+    },
+    {
+      "clip_ratio": 0.000756454566726461,
+      "completion_length": 1099.1250610351562,
+      "epoch": 0.006902663122055874,
+      "grad_norm": 0.07225238531827927,
+      "kl": 0.0067138671875,
+      "learning_rate": 1.251690743723718e-06,
+      "loss": 0.016,
+      "num_tokens": 4282729.0,
+      "reward": 0.1934523843228817,
+      "reward_std": 0.11385035887360573,
+      "rewards/code_reward": 0.10303572192788124,
+      "rewards/format_reward": 0.9041666388511658,
+      "step": 370
+    },
+    {
+      "clip_ratio": 0.000768135127145797,
+      "epoch": 0.006921318968331701,
+      "grad_norm": 0.0729527473449707,
+      "kl": 0.0072021484375,
+      "learning_rate": 1.2408493515534581e-06,
+      "loss": 0.0163,
+      "step": 371
+    },
+    {
+      "clip_ratio": 0.0006400739657692611,
+      "epoch": 0.006939974814607528,
+      "grad_norm": 0.07251974940299988,
+      "kl": 0.007110595703125,
+      "learning_rate": 1.2300712803218834e-06,
+      "loss": 0.0159,
+      "step": 372
+    },
+    {
+      "clip_ratio": 0.0016507117543369532,
+      "completion_length": 431.79168701171875,
+      "epoch": 0.0069586306608833545,
+      "grad_norm": 0.11167676746845245,
+      "kl": 0.014739990234375,
+      "learning_rate": 1.2193569822552772e-06,
+      "loss": -0.0044,
+      "num_tokens": 4303508.0,
+      "reward": 0.47044751420617104,
+      "reward_std": 0.23989063128829002,
+      "rewards/code_reward": 0.3850308656692505,
+      "rewards/format_reward": 0.8541666567325592,
+      "step": 373
+    },
+    {
+      "clip_ratio": 0.001415064267348498,
+      "epoch": 0.006977286507159181,
+      "grad_norm": 0.10820432007312775,
+      "kl": 0.016204833984375,
+      "learning_rate": 1.2087069069041268e-06,
+      "loss": -0.004,
+      "step": 374
+    },
+    {
+      "clip_ratio": 0.0017915331991389394,
+      "epoch": 0.006995942353435008,
+      "grad_norm": 0.10194279253482819,
+      "kl": 0.01751708984375,
+      "learning_rate": 1.1981215011242654e-06,
+      "loss": -0.0037,
+      "step": 375
+    },
+    {
+      "clip_ratio": 0.0009616998431738466,
+      "completion_length": 945.7500610351562,
+      "epoch": 0.007014598199710834,
+      "grad_norm": 0.06673284620046616,
+      "kl": 0.0225830078125,
+      "learning_rate": 1.1876012090581184e-06,
+      "loss": -0.0215,
+      "num_tokens": 4339982.0,
+      "reward": 0.4850490316748619,
+      "reward_std": 0.451216384768486,
+      "rewards/code_reward": 0.39338236302137375,
+      "rewards/format_reward": 0.9166666567325592,
+      "step": 376
+    },
+    {
+      "clip_ratio": 0.0011033907067030668,
+      "epoch": 0.007033254045986661,
+      "grad_norm": 0.06927090138196945,
+      "kl": 0.02490234375,
+      "learning_rate": 1.177146472116071e-06,
+      "loss": -0.0215,
+      "step": 377
+    },
+    {
+      "clip_ratio": 0.0009509723749943078,
+      "epoch": 0.007051909892262488,
+      "grad_norm": 0.06608384847640991,
+      "kl": 0.0247802734375,
+      "learning_rate": 1.1667577289579462e-06,
+      "loss": -0.0216,
+      "step": 378
+    },
+    {
+      "clip_ratio": 0.0002945388841908425,
+      "completion_length": 615.0000152587891,
+      "epoch": 0.007070565738538315,
+      "grad_norm": 0.09600235521793365,
+      "kl": 0.07568359375,
+      "learning_rate": 1.1564354154746007e-06,
+      "loss": 0.0006,
+      "num_tokens": 4370618.0,
+      "reward": 0.5112499818205833,
+      "reward_std": 0.22198906354606152,
+      "rewards/code_reward": 0.4166666567325592,
+      "rewards/format_reward": 0.9458333551883698,
+      "step": 379
+    },
+    {
+      "clip_ratio": 0.0003342999843880534,
+      "epoch": 0.007089221584814141,
+      "grad_norm": 0.09851517528295517,
+      "kl": 0.07568359375,
+      "learning_rate": 1.146179964769635e-06,
+      "loss": 0.0007,
+      "step": 380
+    },
+    {
+      "clip_ratio": 0.00041099853115156293,
+      "epoch": 0.007107877431089968,
+      "grad_norm": 0.10035818070173264,
+      "kl": 0.0843505859375,
+      "learning_rate": 1.1359918071412195e-06,
+      "loss": 0.0008,
+      "step": 381
+    },
+    {
+      "clip_ratio": 0.0005624304030789062,
+      "completion_length": 937.5417175292969,
+      "epoch": 0.007126533277365794,
+      "grad_norm": 0.09549058228731155,
+      "kl": 0.0206298828125,
+      "learning_rate": 1.1258713700640456e-06,
+      "loss": 0.0096,
+      "num_tokens": 4404219.0,
+      "reward": 0.667364239692688,
+      "reward_std": 0.4190048724412918,
+      "rewards/code_reward": 0.576114222407341,
+      "rewards/format_reward": 0.9124999642372131,
+      "step": 382
+    },
+    {
+      "clip_ratio": 0.0006415857642423362,
+      "epoch": 0.007145189123641621,
+      "grad_norm": 0.09549405425786972,
+      "kl": 0.0218505859375,
+      "learning_rate": 1.115819078171383e-06,
+      "loss": 0.0097,
+      "step": 383
+    },
+    {
+      "clip_ratio": 0.0007653232023585588,
+      "epoch": 0.0071638449699174475,
+      "grad_norm": 0.09525733441114426,
+      "kl": 0.021331787109375,
+      "learning_rate": 1.1058353532372667e-06,
+      "loss": 0.0096,
+      "step": 384
+    },
+    {
+      "clip_ratio": 0.000980669748969376,
+      "completion_length": 1172.1666870117188,
+      "epoch": 0.007182500816193275,
+      "grad_norm": 0.07197841256856918,
+      "kl": 0.01171875,
+      "learning_rate": 1.0959206141587998e-06,
+      "loss": -0.0047,
+      "num_tokens": 4448983.0,
+      "reward": 0.11828883737325668,
+      "reward_std": 0.10133323073387146,
+      "rewards/code_reward": 0.05703882873058319,
+      "rewards/format_reward": 0.6124999821186066,
+      "step": 385
+    },
+    {
+      "clip_ratio": 0.0007983763935044408,
+      "epoch": 0.0072011566624691015,
+      "grad_norm": 0.07335319370031357,
+      "kl": 0.01263427734375,
+      "learning_rate": 1.0860752769385766e-06,
+      "loss": -0.0049,
+      "step": 386
+    },
+    {
+      "clip_ratio": 0.001325059012742713,
+      "epoch": 0.007219812508744928,
+      "grad_norm": 0.07985582202672958,
+      "kl": 0.011871337890625,
+      "learning_rate": 1.0762997546672279e-06,
+      "loss": -0.005,
+      "step": 387
+    },
+    {
+      "clip_ratio": 0.0009500840096734464,
+      "completion_length": 764.8750305175781,
+      "epoch": 0.007238468355020755,
+      "grad_norm": 0.103624626994133,
+      "kl": 0.019287109375,
+      "learning_rate": 1.0665944575060914e-06,
+      "loss": 0.0971,
+      "num_tokens": 4478560.0,
+      "reward": 0.47962963581085205,
+      "reward_std": 0.2753693163394928,
+      "rewards/code_reward": 0.383796289563179,
+      "rewards/format_reward": 0.9583333432674408,
+      "step": 388
+    },
+    {
+      "clip_ratio": 0.0007495690369978547,
+      "epoch": 0.007257124201296581,
+      "grad_norm": 0.1009078398346901,
+      "kl": 0.02117919921875,
+      "learning_rate": 1.056959792669997e-06,
+      "loss": 0.0968,
+      "step": 389
+    },
+    {
+      "clip_ratio": 0.0008809762366581708,
+      "epoch": 0.007275780047572408,
+      "grad_norm": 0.10338608175516129,
+      "kl": 0.0205078125,
+      "learning_rate": 1.0473961644101856e-06,
+      "loss": 0.0969,
+      "step": 390
+    },
+    {
+      "clip_ratio": 0.0008085968729574233,
+      "completion_length": 664.7916870117188,
+      "epoch": 0.007294435893848234,
+      "grad_norm": 0.09412232786417007,
+      "kl": 0.02545166015625,
+      "learning_rate": 1.037903973997345e-06,
+      "loss": 0.0,
+      "num_tokens": 4507655.0,
+      "reward": 0.5595833584666252,
+      "reward_std": 0.18141867592930794,
+      "rewards/code_reward": 0.4583333432674408,
+      "rewards/format_reward": 1.012499988079071,
+      "step": 391
+    },
+    {
+      "clip_ratio": 0.0006834749365225434,
+      "epoch": 0.007313091740124062,
+      "grad_norm": 0.13709917664527893,
+      "kl": 0.0255126953125,
+      "learning_rate": 1.0284836197047737e-06,
+      "loss": 0.0001,
+      "step": 392
+    },
+    {
+      "clip_ratio": 0.0006806632154621184,
+      "epoch": 0.007331747586399888,
+      "grad_norm": 0.09327536076307297,
+      "kl": 0.0245361328125,
+      "learning_rate": 1.0191354967916712e-06,
+      "loss": 0.0001,
+      "step": 393
+    },
+    {
+      "clip_ratio": 0.0008892881742212921,
+      "completion_length": 666.8333435058594,
+      "epoch": 0.007350403432675715,
+      "grad_norm": 0.09906277060508728,
+      "kl": 0.02410888671875,
+      "learning_rate": 1.0098599974865515e-06,
+      "loss": 0.0111,
+      "num_tokens": 4542367.0,
+      "reward": 0.771384060382843,
+      "reward_std": 0.17679990455508232,
+      "rewards/code_reward": 0.6901340931653976,
+      "rewards/format_reward": 0.8125,
+      "step": 394
+    },
+    {
+      "clip_ratio": 0.0008279377470898908,
+      "epoch": 0.007369059278951541,
+      "grad_norm": 0.10521873086690903,
+      "kl": 0.02398681640625,
+      "learning_rate": 1.0006575109707898e-06,
+      "loss": 0.0108,
+      "step": 395
+    },
+    {
+      "clip_ratio": 0.0009161636990029365,
+      "epoch": 0.007387715125227368,
+      "grad_norm": 0.09651351720094681,
+      "kl": 0.023193359375,
+      "learning_rate": 9.915284233622877e-07,
+      "loss": 0.011,
+      "step": 396
+    },
+    {
+      "clip_ratio": 0.00039033169741742313,
+      "completion_length": 786.7500457763672,
+      "epoch": 0.0074063709715031945,
+      "grad_norm": 0.09486052393913269,
+      "kl": 0.02703857421875,
+      "learning_rate": 9.824731176992796e-07,
+      "loss": 0.0268,
+      "num_tokens": 4569841.0,
+      "reward": 0.4910014681518078,
+      "reward_std": 0.2589704990386963,
+      "rewards/code_reward": 0.37975146202370524,
+      "rewards/format_reward": 1.1124999523162842,
+      "step": 397
+    },
+    {
+      "clip_ratio": 0.00043224109685979784,
+      "epoch": 0.007425026817779021,
+      "grad_norm": 0.09329359233379364,
+      "kl": 0.03033447265625,
+      "learning_rate": 9.734919739242543e-07,
+      "loss": 0.0269,
+      "step": 398
+    },
+    {
+      "clip_ratio": 0.0005088059406261891,
+      "epoch": 0.0074436826640548485,
+      "grad_norm": 0.09744512289762497,
+      "kl": 0.0316162109375,
+      "learning_rate": 9.645853688680177e-07,
+      "loss": 0.027,
+      "step": 399
+    },
+    {
+      "clip_ratio": 0.0007269750349223614,
+      "completion_length": 853.0833435058594,
+      "epoch": 0.007462338510330675,
+      "grad_norm": 0.07783529907464981,
+      "kl": 0.029083251953125,
+      "learning_rate": 9.557536762338786e-07,
+      "loss": 0.0011,
+      "num_tokens": 4600515.0,
+      "reward": 0.4749346524477005,
+      "reward_std": 0.22702517919242382,
+      "rewards/code_reward": 0.39910128712654114,
+      "rewards/format_reward": 0.7583333253860474,
+      "step": 400
+    },
+    {
+      "clip_ratio": 0.0009095004643313587,
+      "epoch": 0.007480994356606502,
+      "grad_norm": 0.07656848430633545,
+      "kl": 0.029296875,
+      "learning_rate": 9.46997266581973e-07,
+      "loss": 0.0013,
+      "step": 401
+    },
+    {
+      "clip_ratio": 0.0006659157224930823,
+      "epoch": 0.007499650202882328,
+      "grad_norm": 0.07752339541912079,
+      "kl": 0.03289794921875,
+      "learning_rate": 9.383165073137115e-07,
+      "loss": 0.001,
+      "step": 402
+    },
+    {
+      "clip_ratio": 0.0015702053788118064,
+      "completion_length": 578.1666717529297,
+      "epoch": 0.007518306049158155,
+      "grad_norm": 0.11208923161029816,
+      "kl": 0.0478515625,
+      "learning_rate": 9.297117626563687e-07,
+      "loss": -0.0263,
+      "num_tokens": 4624963.0,
+      "reward": 0.8968709111213684,
+      "reward_std": 0.3033272698521614,
+      "rewards/code_reward": 0.78228759765625,
+      "rewards/format_reward": 1.1458333134651184,
+      "step": 403
+    },
+    {
+      "clip_ratio": 0.0014707182417623699,
+      "epoch": 0.007536961895433981,
+      "grad_norm": 0.11273815482854843,
+      "kl": 0.048583984375,
+      "learning_rate": 9.211833936477957e-07,
+      "loss": -0.0262,
+      "step": 404
+    },
+    {
+      "clip_ratio": 0.0014846965787000954,
+      "epoch": 0.007555617741709809,
+      "grad_norm": 0.11112023890018463,
+      "kl": 0.0447998046875,
+      "learning_rate": 9.127317581212753e-07,
+      "loss": -0.0261,
+      "step": 405
+    },
+    {
+      "clip_ratio": 0.000766730576287955,
+      "completion_length": 860.0000305175781,
+      "epoch": 0.007574273587985635,
+      "grad_norm": 0.07397118955850601,
+      "kl": 0.03369140625,
+      "learning_rate": 9.043572106905084e-07,
+      "loss": 0.0091,
+      "num_tokens": 4658083.0,
+      "reward": 0.6170833520591259,
+      "reward_std": 0.16897812485694885,
+      "rewards/code_reward": 0.5416666567325592,
+      "rewards/format_reward": 0.7541666999459267,
+      "step": 406
+    },
+    {
+      "clip_ratio": 0.0006420460704248399,
+      "epoch": 0.007592929434261462,
+      "grad_norm": 0.0758887380361557,
+      "kl": 0.03009033203125,
+      "learning_rate": 8.960601027347321e-07,
+      "loss": 0.0092,
+      "step": 407
+    },
+    {
+      "clip_ratio": 0.0007185917929746211,
+      "epoch": 0.007611585280537288,
+      "grad_norm": 0.07315798103809357,
+      "kl": 0.0299072265625,
+      "learning_rate": 8.878407823839788e-07,
+      "loss": 0.0091,
+      "step": 408
+    },
+    {
+      "clip_ratio": 0.0008533093787264079,
+      "completion_length": 1027.0000305175781,
+      "epoch": 0.007630241126813115,
+      "grad_norm": 0.08570308238267899,
+      "kl": 0.03460693359375,
+      "learning_rate": 8.796995945044689e-07,
+      "loss": 0.0127,
+      "num_tokens": 4699639.0,
+      "reward": 0.06000000238418579,
+      "reward_std": 0.05202655866742134,
+      "rewards/code_reward": 0.0,
+      "rewards/format_reward": 0.5999999940395355,
+      "step": 409
+    },
+    {
+      "clip_ratio": 0.0009294236078858376,
+      "epoch": 0.0076488969730889415,
+      "grad_norm": 0.07877162843942642,
+      "kl": 0.0330810546875,
+      "learning_rate": 8.716368806841405e-07,
+      "loss": 0.0129,
+      "step": 410
+    },
+    {
+      "clip_ratio": 0.0008056100341491401,
+      "epoch": 0.007667552819364768,
+      "grad_norm": 0.07954779267311096,
+      "kl": 0.031005859375,
+      "learning_rate": 8.636529792183171e-07,
+      "loss": 0.0125,
+      "step": 411
+    },
+    {
+      "clip_ratio": 0.0007956986082717776,
+      "completion_length": 737.4166870117188,
+      "epoch": 0.0076862086656405955,
+      "grad_norm": 0.08732157200574875,
+      "kl": 0.027008056640625,
+      "learning_rate": 8.557482250955144e-07,
+      "loss": 0.0307,
+      "num_tokens": 4734041.0,
+      "reward": 0.7656373083591461,
+      "reward_std": 0.2801913693547249,
+      "rewards/code_reward": 0.6764705926179886,
+      "rewards/format_reward": 0.8916666507720947,
+      "step": 412
+    },
+    {
+      "clip_ratio": 0.0005478582752402872,
+      "epoch": 0.007704864511916422,
+      "grad_norm": 0.0853348970413208,
+      "kl": 0.027374267578125,
+      "learning_rate": 8.479229499833844e-07,
+      "loss": 0.0308,
+      "step": 413
+    },
+    {
+      "clip_ratio": 0.0008438964432571083,
+      "epoch": 0.007723520358192249,
+      "grad_norm": 0.08625921607017517,
+      "kl": 0.02655029296875,
+      "learning_rate": 8.401774822147976e-07,
+      "loss": 0.0306,
+      "step": 414
+    },
+    {
+      "clip_ratio": 0.0019030745315831155,
+      "completion_length": 749.3750152587891,
+      "epoch": 0.007742176204468075,
+      "grad_norm": 0.09860255569219589,
+      "kl": 0.041900634765625,
+      "learning_rate": 8.325121467740695e-07,
+      "loss": -0.0314,
+      "num_tokens": 4770374.0,
+      "reward": 0.16708937659859657,
+      "reward_std": 0.18778712674975395,
+      "rewards/code_reward": 0.11292270570993423,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 415
+    },
+    {
+      "clip_ratio": 0.0017334477743133903,
+      "epoch": 0.007760832050743902,
+      "grad_norm": 0.09663218259811401,
+      "kl": 0.039215087890625,
+      "learning_rate": 8.249272652833226e-07,
+      "loss": -0.0314,
+      "step": 416
+    },
+    {
+      "clip_ratio": 0.0020351834245957434,
+      "epoch": 0.007779487897019728,
+      "grad_norm": 0.10066119581460953,
+      "kl": 0.0433349609375,
+      "learning_rate": 8.174231559889931e-07,
+      "loss": -0.0313,
+      "step": 417
+    },
+    {
+      "clip_ratio": 0.0017151011270470917,
+      "completion_length": 631.6666870117188,
+      "epoch": 0.007798143743295555,
+      "grad_norm": 0.09030763059854507,
+      "kl": 0.046661376953125,
+      "learning_rate": 8.100001337484787e-07,
+      "loss": 0.0406,
+      "num_tokens": 4798446.0,
+      "reward": 0.9333169162273407,
+      "reward_std": 0.1303430050611496,
+      "rewards/code_reward": 0.8399835228919983,
+      "rewards/format_reward": 0.9333333075046539,
+      "step": 418
+    },
+    {
+      "clip_ratio": 0.0014318364555947483,
+      "epoch": 0.007816799589571382,
+      "grad_norm": 0.1113353744149208,
+      "kl": 0.051666259765625,
+      "learning_rate": 8.026585100169251e-07,
+      "loss": 0.0409,
+      "step": 419
+    },
+    {
+      "clip_ratio": 0.0014859928633086383,
+      "epoch": 0.007835455435847209,
+      "grad_norm": 0.09183201938867569,
+      "kl": 0.055267333984375,
+      "learning_rate": 7.953985928341601e-07,
+      "loss": 0.0409,
+      "step": 420
+    },
+    {
+      "clip_ratio": 0.0009874806564766914,
+      "completion_length": 722.5833435058594,
+      "epoch": 0.007854111282123035,
+      "grad_norm": 0.301491379737854,
+      "kl": 0.03594970703125,
+      "learning_rate": 7.882206868117693e-07,
+      "loss": 0.0123,
+      "num_tokens": 4831364.0,
+      "reward": 0.46648313850164413,
+      "reward_std": 0.24989983439445496,
+      "rewards/code_reward": 0.39314980432391167,
+      "rewards/format_reward": 0.7333333492279053,
+      "step": 421
+    },
+    {
+      "clip_ratio": 0.0007859274046495557,
+      "epoch": 0.007872767128398862,
+      "grad_norm": 0.09945830702781677,
+      "kl": 0.03179931640625,
+      "learning_rate": 7.81125093120313e-07,
+      "loss": 0.0124,
+      "step": 422
+    },
+    {
+      "clip_ratio": 0.0010357568971812725,
+      "epoch": 0.007891422974674689,
+      "grad_norm": 0.09847865253686905,
+      "kl": 0.03509521484375,
+      "learning_rate": 7.741121094766916e-07,
+      "loss": 0.0124,
+      "step": 423
+    },
+    {
+      "clip_ratio": 0.0015808032476343215,
+      "completion_length": 677.6250152587891,
+      "epoch": 0.007910078820950515,
+      "grad_norm": 0.1073075532913208,
+      "kl": 0.05718994140625,
+      "learning_rate": 7.671820301316532e-07,
+      "loss": 0.0359,
+      "num_tokens": 4852811.0,
+      "reward": 0.2411028817296028,
+      "reward_std": 0.08415344171226025,
+      "rewards/code_reward": 0.15360287349903956,
+      "rewards/format_reward": 0.875,
+      "step": 424
+    },
+    {
+      "clip_ratio": 0.0017813361482694745,
+      "epoch": 0.007928734667226342,
+      "grad_norm": 0.10579974949359894,
+      "kl": 0.05438232421875,
+      "learning_rate": 7.603351458574474e-07,
+      "loss": 0.0363,
+      "step": 425
+    },
+    {
+      "clip_ratio": 0.001651192782446742,
+      "epoch": 0.007947390513502168,
+      "grad_norm": 0.11324102431535721,
+      "kl": 0.05828857421875,
+      "learning_rate": 7.535717439356255e-07,
+      "loss": 0.0361,
+      "step": 426
+    },
+    {
+      "clip_ratio": 0.0014180371072143316,
+      "completion_length": 693.3333435058594,
+      "epoch": 0.007966046359777995,
+      "grad_norm": 0.16692698001861572,
+      "kl": 0.0213623046875,
+      "learning_rate": 7.46892108144986e-07,
+      "loss": 0.0654,
+      "num_tokens": 4883755.0,
+      "reward": 0.6448469460010529,
+      "reward_std": 0.49255189299583435,
+      "rewards/code_reward": 0.5340135842561722,
+      "rewards/format_reward": 1.1083332896232605,
+      "step": 427
+    },
+    {
+      "clip_ratio": 0.0013092211447656155,
+      "epoch": 0.007984702206053821,
+      "grad_norm": 0.09344299882650375,
+      "kl": 0.02069091796875,
+      "learning_rate": 7.402965187496697e-07,
+      "loss": 0.0651,
+      "step": 428
+    },
+    {
+      "clip_ratio": 0.0009931237436830997,
+      "epoch": 0.00800335805232965,
+      "grad_norm": 0.0950753316283226,
+      "kl": 0.02081298828125,
+      "learning_rate": 7.337852524873974e-07,
+      "loss": 0.0652,
+      "step": 429
+    },
+    {
+      "clip_ratio": 0.000936219235882163,
+      "completion_length": 608.0000152587891,
+      "epoch": 0.008022013898605476,
+      "grad_norm": 0.10651998966932297,
+      "kl": 0.080657958984375,
+      "learning_rate": 7.273585825578608e-07,
+      "loss": -0.0054,
+      "num_tokens": 4910095.0,
+      "reward": 0.6445792466402054,
+      "reward_std": 0.16696248203516006,
+      "rewards/code_reward": 0.5420792102813721,
+      "rewards/format_reward": 1.024999976158142,
+      "step": 430
+    },
+    {
+      "clip_ratio": 0.0005899647367186844,
+      "epoch": 0.008040669744881303,
+      "grad_norm": 0.10619425773620605,
+      "kl": 0.083984375,
+      "learning_rate": 7.21016778611259e-07,
+      "loss": -0.0052,
+      "step": 431
+    },
+    {
+      "clip_ratio": 0.0008555436506867409,
+      "epoch": 0.00805932559115713,
+      "grad_norm": 0.10411829501390457,
+      "kl": 0.0823974609375,
+      "learning_rate": 7.147601067369835e-07,
+      "loss": -0.0054,
+      "step": 432
+    },
+    {
+      "clip_ratio": 0.0007757125422358513,
+      "completion_length": 393.5833435058594,
+      "epoch": 0.008077981437432956,
+      "grad_norm": 0.11761093884706497,
+      "kl": 0.08935546875,
+      "learning_rate": 7.085888294524561e-07,
+      "loss": -0.0138,
+      "num_tokens": 4927329.0,
+      "reward": 0.6537500023841858,
+      "reward_std": 0.17917166277766228,
+      "rewards/code_reward": 0.5416666679084301,
+      "rewards/format_reward": 1.120833307504654,
+      "step": 433
+    },
+    {
+      "clip_ratio": 0.0007667961181141436,
+      "epoch": 0.008096637283708782,
+      "grad_norm": 0.11278140544891357,
+      "kl": 0.076904296875,
+      "learning_rate": 7.025032056921117e-07,
+      "loss": -0.0137,
+      "step": 434
+    },
+    {
+      "clip_ratio": 0.000893244257895276,
+      "epoch": 0.008115293129984609,
+      "grad_norm": 0.1142934262752533,
+      "kl": 0.077880859375,
+      "learning_rate": 6.965034907965349e-07,
+      "loss": -0.0139,
+      "step": 435
+    },
+    {
+      "clip_ratio": 0.0005022610421292484,
+      "completion_length": 807.9583435058594,
+      "epoch": 0.008133948976260436,
+      "grad_norm": 0.0828719511628151,
+      "kl": 0.0184326171875,
+      "learning_rate": 6.905899365017462e-07,
+      "loss": 0.027,
+      "num_tokens": 4961552.0,
+      "reward": 0.7701181471347809,
+      "reward_std": 0.3888111412525177,
+      "rewards/code_reward": 0.6847014874219894,
+      "rewards/format_reward": 0.8541666865348816,
+      "step": 436
+    },
+    {
+      "clip_ratio": 0.0007263242732733488,
+      "epoch": 0.008152604822536262,
+      "grad_norm": 0.08436731994152069,
+      "kl": 0.0183563232421875,
+      "learning_rate": 6.847627909286409e-07,
+      "loss": 0.0272,
+      "step": 437
+    },
+    {
+      "clip_ratio": 0.0007911297434475273,
+      "epoch": 0.008171260668812089,
+      "grad_norm": 0.08241833746433258,
+      "kl": 0.0186309814453125,
+      "learning_rate": 6.790222985725761e-07,
+      "loss": 0.027,
+      "step": 438
+    },
+    {
+      "clip_ratio": 0.0008176612027455121,
+      "completion_length": 741.7083435058594,
+      "epoch": 0.008189916515087915,
+      "grad_norm": 0.09797824174165726,
+      "kl": 0.02587890625,
+      "learning_rate": 6.733687002931141e-07,
+      "loss": -0.0261,
+      "num_tokens": 4989097.0,
+      "reward": 0.3370833322405815,
+      "reward_std": 0.2753293476998806,
+      "rewards/code_reward": 0.2291666716337204,
+      "rewards/format_reward": 1.0791666209697723,
+      "step": 439
+    },
+    {
+      "clip_ratio": 0.0010906177922151983,
+      "epoch": 0.008208572361363742,
+      "grad_norm": 0.10210465639829636,
+      "kl": 0.026519775390625,
+      "learning_rate": 6.678022333039158e-07,
+      "loss": -0.0261,
+      "step": 440
+    },
+    {
+      "clip_ratio": 0.0006749743188265711,
+      "epoch": 0.008227228207639568,
+      "grad_norm": 0.09442983567714691,
+      "kl": 0.027008056640625,
+      "learning_rate": 6.623231311627876e-07,
+      "loss": -0.0263,
+      "step": 441
+    },
+    {
+      "clip_ratio": 0.0011484751594252884,
+      "completion_length": 866.75,
+      "epoch": 0.008245884053915395,
+      "grad_norm": 0.07345132529735565,
+      "kl": 0.03179931640625,
+      "learning_rate": 6.569316237618811e-07,
+      "loss": 0.0639,
+      "num_tokens": 5026687.0,
+      "reward": 0.2147694230079651,
+      "reward_std": 0.33931225538253784,
+      "rewards/code_reward": 0.132686085999012,
+      "rewards/format_reward": 0.8208333253860474,
+      "step": 442
+    },
+    {
+      "clip_ratio": 0.0010752396774478257,
+      "epoch": 0.008264539900191223,
+      "grad_norm": 0.07625681161880493,
+      "kl": 0.0343017578125,
+      "learning_rate": 6.516279373180499e-07,
+      "loss": 0.0637,
+      "step": 443
+    },
+    {
+      "clip_ratio": 0.001316602574661374,
+      "epoch": 0.00828319574646705,
+      "grad_norm": 0.07411348074674606,
+      "kl": 0.0333251953125,
+      "learning_rate": 6.464122943633543e-07,
+      "loss": 0.0638,
+      "step": 444
+    },
+    {
+      "clip_ratio": 0.0007968830032041296,
+      "completion_length": 739.5000305175781,
+      "epoch": 0.008301851592742876,
+      "grad_norm": 0.09090346097946167,
+      "kl": 0.0362548828125,
+      "learning_rate": 6.412849137357271e-07,
+      "loss": 0.0124,
+      "num_tokens": 5059039.0,
+      "reward": 0.6221906468272209,
+      "reward_std": 0.09545084834098816,
+      "rewards/code_reward": 0.530940592288971,
+      "rewards/format_reward": 0.9124999940395355,
+      "step": 445
+    },
+    {
+      "clip_ratio": 0.0005075681256130338,
+      "epoch": 0.008320507439018703,
+      "grad_norm": 0.09324011206626892,
+      "kl": 0.0374755859375,
+      "learning_rate": 6.3624601056979e-07,
+      "loss": 0.0122,
+      "step": 446
+    },
+    {
+      "clip_ratio": 0.0002910211042035371,
+      "epoch": 0.00833916328529453,
+      "grad_norm": 0.0913727805018425,
+      "kl": 0.0367431640625,
+      "learning_rate": 6.312957962878278e-07,
+      "loss": 0.0122,
+      "step": 447
+    },
+    {
+      "clip_ratio": 0.0009194318408844993,
+      "completion_length": 722.7500305175781,
+      "epoch": 0.008357819131570356,
+      "grad_norm": 0.08921171724796295,
+      "kl": 0.015167236328125,
+      "learning_rate": 6.264344785909181e-07,
+      "loss": -0.0187,
+      "num_tokens": 5088301.0,
+      "reward": 0.3760389983654022,
+      "reward_std": 0.21540092676877975,
+      "rewards/code_reward": 0.25312230736017227,
+      "rewards/format_reward": 1.2291666865348816,
+      "step": 448
+    },
+    {
+      "clip_ratio": 0.000731065621948801,
+      "epoch": 0.008376474977846183,
+      "grad_norm": 0.08806627243757248,
+      "kl": 0.0145263671875,
+      "learning_rate": 6.216622614502149e-07,
+      "loss": -0.0183,
+      "step": 449
+    },
+    {
+      "clip_ratio": 0.001043492229655385,
+      "epoch": 0.008395130824122009,
+      "grad_norm": 0.08825229853391647,
+      "kl": 0.013092041015625,
+      "learning_rate": 6.169793450983916e-07,
+      "loss": -0.0185,
+      "step": 450
+    },
+    {
+      "clip_ratio": 0.0011616212141234428,
+      "completion_length": 723.0416870117188,
+      "epoch": 0.008413786670397836,
+      "grad_norm": 0.10913647711277008,
+      "kl": 0.0169677734375,
+      "learning_rate": 6.123859260212393e-07,
+      "loss": 0.0171,
+      "num_tokens": 5114834.0,
+      "reward": 0.4591666907072067,
+      "reward_std": 0.2421144973486662,
+      "rewards/code_reward": 0.375,
+      "rewards/format_reward": 0.8416666686534882,
+      "step": 451
+    },
+    {
+      "clip_ratio": 0.0010601775429677218,
+      "epoch": 0.008432442516673662,
+      "grad_norm": 0.1069192886352539,
+      "kl": 0.0181884765625,
+      "learning_rate": 6.07882196949423e-07,
+      "loss": 0.0172,
+      "step": 452
+    },
+    {
+      "clip_ratio": 0.0011184008326381445,
+      "epoch": 0.008451098362949489,
+      "grad_norm": 0.10386621952056885,
+      "kl": 0.017730712890625,
+      "learning_rate": 6.034683468503948e-07,
+      "loss": 0.0172,
+      "step": 453
+    },
+    {
+      "clip_ratio": 0.001289009174797684,
+      "completion_length": 979.8333740234375,
+      "epoch": 0.008469754209225315,
+      "grad_norm": 0.08415919542312622,
+      "kl": 0.011962890625,
+      "learning_rate": 5.991445609204641e-07,
+      "loss": -0.0303,
+      "num_tokens": 5149462.0,
+      "reward": 0.5253205597400665,
+      "reward_std": 0.1456993781030178,
+      "rewards/code_reward": 0.4149038642644882,
+      "rewards/format_reward": 1.1041666269302368,
+      "step": 454
+    },
+    {
+      "clip_ratio": 0.000828628777526319,
+      "epoch": 0.008488410055501142,
+      "grad_norm": 0.0916297659277916,
+      "kl": 0.011932373046875,
+      "learning_rate": 5.949110205770292e-07,
+      "loss": -0.0305,
+      "step": 455
+    },
+    {
+      "clip_ratio": 0.0010398888844065368,
+      "epoch": 0.00850706590177697,
+      "grad_norm": 0.08343882113695145,
+      "kl": 0.0126953125,
+      "learning_rate": 5.90767903450964e-07,
+      "loss": -0.0306,
+      "step": 456
+    },
+    {
+      "clip_ratio": 0.0010439236648380756,
+      "completion_length": 754.125,
+      "epoch": 0.008525721748052797,
+      "grad_norm": 0.08621830493211746,
+      "kl": 0.019927978515625,
+      "learning_rate": 5.867153833791652e-07,
+      "loss": 0.0192,
+      "num_tokens": 5181685.0,
+      "reward": 0.3584918677806854,
+      "reward_std": 0.23101448267698288,
+      "rewards/code_reward": 0.2543252221075818,
+      "rewards/format_reward": 1.0416666567325592,
+      "step": 457
+    },
+    {
+      "clip_ratio": 0.0011347323888912797,
+      "epoch": 0.008544377594328623,
+      "grad_norm": 0.08586623519659042,
+      "kl": 0.020263671875,
+      "learning_rate": 5.827536303972587e-07,
+      "loss": 0.0193,
+      "step": 458
+    },
+    {
+      "clip_ratio": 0.0011065362195950001,
+      "epoch": 0.00856303344060445,
+      "grad_norm": 0.08708977699279785,
+      "kl": 0.018890380859375,
+      "learning_rate": 5.78882810732465e-07,
+      "loss": 0.0192,
+      "step": 459
+    },
+    {
+      "clip_ratio": 0.0007787059585098177,
+      "completion_length": 975.9167175292969,
+      "epoch": 0.008581689286880276,
+      "grad_norm": 0.11942039430141449,
+      "kl": 0.025390625,
+      "learning_rate": 5.75103086796625e-07,
+      "loss": 0.0441,
+      "num_tokens": 5223179.0,
+      "reward": 0.0820833370089531,
+      "reward_std": 0.06146477162837982,
+      "rewards/code_reward": 0.0,
+      "rewards/format_reward": 0.8208333253860474,
+      "step": 460
+    },
+    {
+      "clip_ratio": 0.0007732175581622869,
+      "epoch": 0.008600345133156103,
+      "grad_norm": 0.0921056792140007,
+      "kl": 0.02691650390625,
+      "learning_rate": 5.714146171793846e-07,
+      "loss": 0.0444,
+      "step": 461
+    },
+    {
+      "clip_ratio": 0.0012547720107249916,
+      "epoch": 0.00861900097943193,
+      "grad_norm": 0.09074374288320541,
+      "kl": 0.02606201171875,
+      "learning_rate": 5.678175566415422e-07,
+      "loss": 0.0445,
+      "step": 462
+    },
+    {
+      "clip_ratio": 0.0012275170302018523,
+      "completion_length": 753.1666870117188,
+      "epoch": 0.008637656825707756,
+      "grad_norm": 0.10298697650432587,
+      "kl": 0.0291748046875,
+      "learning_rate": 5.643120561085528e-07,
+      "loss": -0.0121,
+      "num_tokens": 5257791.0,
+      "reward": 0.3049456551671028,
+      "reward_std": 0.2707311660051346,
+      "rewards/code_reward": 0.22036230191588402,
+      "rewards/format_reward": 0.8458333015441895,
+      "step": 463
+    },
+    {
+      "clip_ratio": 0.0011390139115974307,
+      "epoch": 0.008656312671983583,
+      "grad_norm": 0.10827159136533737,
+      "kl": 0.02630615234375,
+      "learning_rate": 5.608982626641991e-07,
+      "loss": -0.0124,
+      "step": 464
+    },
+    {
+      "clip_ratio": 0.001239880220964551,
+      "epoch": 0.00867496851825941,
+      "grad_norm": 0.10058361291885376,
+      "kl": 0.0302734375,
+      "learning_rate": 5.575763195444166e-07,
+      "loss": -0.0123,
+      "step": 465
+    },
+    {
+      "clip_ratio": 0.0006581109191756696,
+      "completion_length": 424.9583435058594,
+      "epoch": 0.008693624364535236,
+      "grad_norm": 0.1420270800590515,
+      "kl": 0.0567169189453125,
+      "learning_rate": 5.543463661312847e-07,
+      "loss": -0.0107,
+      "num_tokens": 5276846.0,
+      "reward": 0.6923423409461975,
+      "reward_std": 0.068264564499259,
+      "rewards/code_reward": 0.5506756789982319,
+      "rewards/format_reward": 1.4166666269302368,
+      "step": 466
+    },
+    {
+      "clip_ratio": 0.0005906374426558614,
+      "epoch": 0.008712280210811062,
+      "grad_norm": 0.13097578287124634,
+      "kl": 0.0464019775390625,
+      "learning_rate": 5.512085379471808e-07,
+      "loss": -0.0104,
+      "step": 467
+    },
+    {
+      "clip_ratio": 0.0008538902620784938,
+      "epoch": 0.008730936057086889,
+      "grad_norm": 0.13688351213932037,
+      "kl": 0.0464019775390625,
+      "learning_rate": 5.481629666490903e-07,
+      "loss": -0.0102,
+      "step": 468
+    },
+    {
+      "clip_ratio": 0.0010946058901026845,
+      "completion_length": 817.5833435058594,
+      "epoch": 0.008749591903362716,
+      "grad_norm": 0.09143988788127899,
+      "kl": 0.03515625,
+      "learning_rate": 5.452097800230853e-07,
+      "loss": -0.0014,
+      "num_tokens": 5311588.0,
+      "reward": 0.29928937181830406,
+      "reward_std": 0.14651047810912132,
+      "rewards/code_reward": 0.21553937532007694,
+      "rewards/format_reward": 0.8374999761581421,
+      "step": 469
+    },
+    {
+      "clip_ratio": 0.0012111273827031255,
+      "epoch": 0.008768247749638544,
+      "grad_norm": 0.10869612544775009,
+      "kl": 0.03814697265625,
+      "learning_rate": 5.423491019789623e-07,
+      "loss": -0.0015,
+      "step": 470
+    },
+    {
+      "clip_ratio": 0.0006720305973431095,
+      "epoch": 0.00878690359591437,
+      "grad_norm": 0.10290330648422241,
+      "kl": 0.03741455078125,
+      "learning_rate": 5.395810525450425e-07,
+      "loss": -0.0014,
+      "step": 471
+    },
+    {
+      "clip_ratio": 0.0014231341774575412,
+      "completion_length": 797.8333740234375,
+      "epoch": 0.008805559442190197,
+      "grad_norm": 0.09828333556652069,
+      "kl": 0.0361328125,
+      "learning_rate": 5.369057478631359e-07,
+      "loss": 0.0535,
+      "num_tokens": 5343864.0,
+      "reward": 0.22396931797266006,
+      "reward_std": 0.12565985321998596,
+      "rewards/code_reward": 0.12938596308231354,
+      "rewards/format_reward": 0.945833295583725,
+      "step": 472
+    },
+    {
+      "clip_ratio": 0.0012132310948800296,
+      "epoch": 0.008824215288466023,
+      "grad_norm": 0.10423380881547928,
+      "kl": 0.0355224609375,
+      "learning_rate": 5.343233001836694e-07,
+      "loss": 0.0532,
+      "step": 473
+    },
+    {
+      "clip_ratio": 0.001151920121628791,
+      "epoch": 0.00884287113474185,
+      "grad_norm": 0.10031603276729584,
+      "kl": 0.03497314453125,
+      "learning_rate": 5.318338178609754e-07,
+      "loss": 0.0533,
+      "step": 474
+    },
+    {
+      "clip_ratio": 0.001284417463466525,
+      "completion_length": 560.7083435058594,
+      "epoch": 0.008861526981017677,
+      "grad_norm": 0.10862758755683899,
+      "kl": 0.05657958984375,
+      "learning_rate": 5.294374053487459e-07,
+      "loss": -0.0221,
+      "num_tokens": 5366837.0,
+      "reward": 0.7270297110080719,
+      "reward_std": 0.4961380064487457,
+      "rewards/code_reward": 0.6303630471229553,
+      "rewards/format_reward": 0.9666666388511658,
+      "step": 475
+    },
+    {
+      "clip_ratio": 0.0007715776737313718,
+      "epoch": 0.008880182827293503,
+      "grad_norm": 0.10750824958086014,
+      "kl": 0.05804443359375,
+      "learning_rate": 5.271341631956511e-07,
+      "loss": -0.0221,
+      "step": 476
+    },
+    {
+      "clip_ratio": 0.0011225882335565984,
+      "epoch": 0.00889883867356933,
+      "grad_norm": 0.10623571276664734,
+      "kl": 0.05712890625,
+      "learning_rate": 5.249241880411181e-07,
+      "loss": -0.0221,
+      "step": 477
+    },
+    {
+      "clip_ratio": 0.0007046593527775258,
+      "completion_length": 761.6250305175781,
+      "epoch": 0.008917494519845156,
+      "grad_norm": 0.10407714545726776,
+      "kl": 0.0296630859375,
+      "learning_rate": 5.228075726112785e-07,
+      "loss": 0.0165,
+      "num_tokens": 5401616.0,
+      "reward": 0.28155867755413055,
+      "reward_std": 0.2833222597837448,
+      "rewards/code_reward": 0.18364198692142963,
+      "rewards/format_reward": 0.9791666567325592,
+      "step": 478
+    },
+    {
+      "clip_ratio": 0.0008142671431414783,
+      "epoch": 0.008936150366120983,
+      "grad_norm": 0.10599526017904282,
+      "kl": 0.02685546875,
+      "learning_rate": 5.207844057150768e-07,
+      "loss": 0.0164,
+      "step": 479
+    },
+    {
+      "clip_ratio": 0.0007827770750736818,
+      "epoch": 0.00895480621239681,
+      "grad_norm": 0.10547417402267456,
+      "kl": 0.02947998046875,
+      "learning_rate": 5.188547722405437e-07,
+      "loss": 0.0163,
+      "step": 480
+    },
+    {
+      "clip_ratio": 0.0009513153345324099,
+      "completion_length": 1125.7500305175781,
+      "epoch": 0.008973462058672636,
+      "grad_norm": 0.10954833775758743,
+      "kl": 0.014617919921875,
+      "learning_rate": 5.170187531512351e-07,
+      "loss": 0.0535,
+      "num_tokens": 5448530.0,
+      "reward": 0.08833334222435951,
+      "reward_std": 0.054226743057370186,
+      "rewards/code_reward": 0.0,
+      "rewards/format_reward": 0.8833333253860474,
+      "step": 481
+    },
+    {
+      "clip_ratio": 0.0012509520165622234,
+      "epoch": 0.008992117904948463,
+      "grad_norm": 0.09105175733566284,
+      "kl": 0.01336669921875,
+      "learning_rate": 5.152764254828348e-07,
+      "loss": 0.0535,
+      "step": 482
+    },
+    {
+      "clip_ratio": 0.0008532060019206256,
+      "epoch": 0.009010773751224289,
+      "grad_norm": 0.09203584492206573,
+      "kl": 0.01513671875,
+      "learning_rate": 5.136278623399225e-07,
+      "loss": 0.0533,
+      "step": 483
+    },
+    {
+      "clip_ratio": 0.0007692791696172208,
+      "completion_length": 997.0833740234375,
+      "epoch": 0.009029429597500117,
+      "grad_norm": 0.09475655108690262,
+      "kl": 0.03662109375,
+      "learning_rate": 5.120731328929058e-07,
+      "loss": -0.0121,
+      "num_tokens": 5492116.0,
+      "reward": 0.5058333426713943,
+      "reward_std": 0.3466748893260956,
+      "rewards/code_reward": 0.40416665747761726,
+      "rewards/format_reward": 1.0166666209697723,
+      "step": 484
+    },
+    {
+      "clip_ratio": 0.000924331892747432,
+      "epoch": 0.009048085443775944,
+      "grad_norm": 0.09625120460987091,
+      "kl": 0.033935546875,
+      "learning_rate": 5.106123023751187e-07,
+      "loss": -0.0121,
+      "step": 485
+    },
+    {
+      "clip_ratio": 0.0005081451236037537,
+      "epoch": 0.00906674129005177,
+      "grad_norm": 0.08786183595657349,
+      "kl": 0.03375244140625,
+      "learning_rate": 5.092454320800833e-07,
+      "loss": -0.0123,
+      "step": 486
+    },
+    {
+      "clip_ratio": 0.00082587351789698,
+      "completion_length": 691.6250305175781,
+      "epoch": 0.009085397136327597,
+      "grad_norm": 0.08786230534315109,
+      "kl": 0.0277099609375,
+      "learning_rate": 5.079725793589405e-07,
+      "loss": 0.0248,
+      "num_tokens": 5518999.0,
+      "reward": 0.9639226198196411,
+      "reward_std": 0.33985109627246857,
+      "rewards/code_reward": 0.861005961894989,
+      "rewards/format_reward": 1.0291666388511658,
+      "step": 487
+    },
+    {
+      "clip_ratio": 0.0007328885840252042,
+      "epoch": 0.009104052982603424,
+      "grad_norm": 0.08928890526294708,
+      "kl": 0.02886962890625,
+      "learning_rate": 5.067937976180407e-07,
+      "loss": 0.0248,
+      "step": 488
+    },
+    {
+      "clip_ratio": 0.0006209511193446815,
+      "epoch": 0.00912270882887925,
+      "grad_norm": 0.08875181525945663,
+      "kl": 0.02874755859375,
+      "learning_rate": 5.057091363167046e-07,
+      "loss": 0.0247,
+      "step": 489
+    },
+    {
+      "clip_ratio": 0.0006376176825142466,
+      "completion_length": 884.0000610351562,
+      "epoch": 0.009141364675155077,
+      "grad_norm": 0.0868891254067421,
+      "kl": 0.0316162109375,
+      "learning_rate": 5.047186409651489e-07,
+      "loss": 0.0218,
+      "num_tokens": 5557063.0,
+      "reward": 0.3246118724346161,
+      "reward_std": 0.20474953204393387,
+      "rewards/code_reward": 0.25627854466438293,
+      "rewards/format_reward": 0.6833333075046539,
+      "step": 490
+    },
+    {
+      "clip_ratio": 0.0008357224578503519,
+      "epoch": 0.009160020521430903,
+      "grad_norm": 0.08658529072999954,
+      "kl": 0.03106689453125,
+      "learning_rate": 5.038223531225742e-07,
+      "loss": 0.022,
+      "step": 491
+    },
+    {
+      "clip_ratio": 0.0008412750175921246,
+      "epoch": 0.00917867636770673,
+      "grad_norm": 0.08690661936998367,
+      "kl": 0.0299072265625,
+      "learning_rate": 5.030203103954232e-07,
+      "loss": 0.022,
+      "step": 492
+    },
+    {
+      "clip_ratio": 0.000971353001659736,
+      "completion_length": 961.7916870117188,
+      "epoch": 0.009197332213982556,
+      "grad_norm": 0.08556245267391205,
+      "kl": 0.019775390625,
+      "learning_rate": 5.023125464358026e-07,
+      "loss": -0.0011,
+      "num_tokens": 5594090.0,
+      "reward": 0.2715613692998886,
+      "reward_std": 0.2223658785223961,
+      "rewards/code_reward": 0.18156136944890022,
+      "rewards/format_reward": 0.9000000357627869,
+      "step": 493
+    },
+    {
+      "clip_ratio": 0.0009858837001957,
+      "epoch": 0.009215988060258383,
+      "grad_norm": 0.08375484496355057,
+      "kl": 0.01922607421875,
+      "learning_rate": 5.016990909400709e-07,
+      "loss": -0.0013,
+      "step": 494
+    },
+    {
+      "clip_ratio": 0.000899194972589612,
+      "epoch": 0.00923464390653421,
+      "grad_norm": 0.08476914465427399,
+      "kl": 0.02081298828125,
+      "learning_rate": 5.011799696475915e-07,
+      "loss": -0.0011,
+      "step": 495
+    },
+    {
+      "clip_ratio": 0.0013499458436854184,
+      "completion_length": 723.0000305175781,
+      "epoch": 0.009253299752810036,
+      "grad_norm": 0.11245770007371902,
+      "kl": 0.0792236328125,
+      "learning_rate": 5.007552043396547e-07,
+      "loss": 0.0186,
+      "num_tokens": 5628218.0,
+      "reward": 0.19834937155246735,
+      "reward_std": 0.12879429571330547,
+      "rewards/code_reward": 0.11418268829584122,
+      "rewards/format_reward": 0.8416666686534882,
+      "step": 496
+    },
+    {
+      "clip_ratio": 0.0017962601850740612,
+      "epoch": 0.009271955599085864,
+      "grad_norm": 0.1085243970155716,
+      "kl": 0.07861328125,
+      "learning_rate": 5.004248128385618e-07,
+      "loss": 0.019,
+      "step": 497
+    },
+    {
+      "clip_ratio": 0.001379593217279762,
+      "epoch": 0.009290611445361691,
+      "grad_norm": 0.110112763941288,
+      "kl": 0.0782470703125,
+      "learning_rate": 5.001888090068784e-07,
+      "loss": 0.0188,
+      "step": 498
+    },
+    {
+      "clip_ratio": 0.0009661116928327829,
+      "completion_length": 820.2083435058594,
+      "epoch": 0.009309267291637517,
+      "grad_norm": 0.09085725247859955,
+      "kl": 0.030242919921875,
+      "learning_rate": 5.000472027468528e-07,
+      "loss": 0.0247,
+      "num_tokens": 5666887.0,
+      "reward": 0.0845833346247673,
+      "reward_std": 0.05373080633580685,
+      "rewards/code_reward": 0.0,
+      "rewards/format_reward": 0.8458333313465118,
+      "step": 499
+    },
+    {
+      "clip_ratio": 0.0010680440173018724,
+      "epoch": 0.009327923137913344,
+      "grad_norm": 0.08862722665071487,
+      "kl": 0.02752685546875,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.0244,
+      "step": 500
+    },
+    {
+      "epoch": 0.009327923137913344,
+      "step": 500,
+      "total_flos": 0.0,
+      "train_loss": 0.010072903922897239,
+      "train_runtime": 28596.067,
+      "train_samples_per_second": 0.42,
+      "train_steps_per_second": 0.017
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}