diff --git "a/checkpoint-1000/trainer_state.json" "b/checkpoint-1000/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-1000/trainer_state.json"
@@ -0,0 +1,19034 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.2,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0002,
+      "grad_norm": 0.4551391005516052,
+      "kl": 0.0006267073404160328,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "num_tokens": 8600.0,
+      "reward": 0.7076416015625,
+      "reward_std": 0.014151658862829208,
+      "rewards//mean": 0.7076416015625,
+      "rewards//std": 0.0565522275865078,
+      "step": 1
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0004,
+      "grad_norm": 0.4617377817630768,
+      "kl": 0.0006313717240118422,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 0.0,
+      "num_tokens": 17200.0,
+      "reward": 0.72869873046875,
+      "reward_std": 0.012252680957317352,
+      "rewards//mean": 0.72869873046875,
+      "rewards//std": 0.06718378514051437,
+      "step": 2
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0006,
+      "grad_norm": 0.47510647773742676,
+      "kl": 0.0007176821964094415,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 0.0,
+      "num_tokens": 25872.0,
+      "reward": 0.7418212890625,
+      "reward_std": 0.01208210177719593,
+      "rewards//mean": 0.7418212890625,
+      "rewards//std": 0.05206342041492462,
+      "step": 3
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0008,
+      "grad_norm": 0.4658019244670868,
+      "kl": 0.0007370488237938844,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 0.0,
+      "num_tokens": 34600.0,
+      "reward": 0.70538330078125,
+      "reward_std": 0.015659615397453308,
+      "rewards//mean": 0.70538330078125,
+      "rewards//std": 0.05750183388590813,
+      "step": 4
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.001,
+      "grad_norm": 0.4073522984981537,
+      "kl": 0.0006538679663208313,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 0.0,
+      "num_tokens": 43304.0,
+      "reward": 0.71893310546875,
+      "reward_std": 0.014116497710347176,
+      "rewards//mean": 0.71893310546875,
+      "rewards//std": 0.06529892981052399,
+      "step": 5
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0012,
+      "grad_norm": 0.45524272322654724,
+      "kl": 0.0006982390259508975,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.0,
+      "num_tokens": 51992.0,
+      "reward": 0.71209716796875,
+      "reward_std": 0.017292022705078125,
+      "rewards//mean": 0.71209716796875,
+      "rewards//std": 0.060261402279138565,
+      "step": 6
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0014,
+      "grad_norm": 0.44682666659355164,
+      "kl": 0.0009362539058201946,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 0.0,
+      "num_tokens": 60696.0,
+      "reward": 0.7254638671875,
+      "reward_std": 0.014938775449991226,
+      "rewards//mean": 0.7254638671875,
+      "rewards//std": 0.04342832788825035,
+      "step": 7
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0016,
+      "grad_norm": 0.453555703163147,
+      "kl": 0.0006852814112789929,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 0.0,
+      "num_tokens": 69336.0,
+      "reward": 0.72991943359375,
+      "reward_std": 0.01723172701895237,
+      "rewards//mean": 0.72991943359375,
+      "rewards//std": 0.05862419679760933,
+      "step": 8
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0018,
+      "grad_norm": 0.49192875623703003,
+      "kl": 0.000731307256501168,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 0.0,
+      "num_tokens": 78008.0,
+      "reward": 0.70782470703125,
+      "reward_std": 0.018347103148698807,
+      "rewards//mean": 0.70782470703125,
+      "rewards//std": 0.058675043284893036,
+      "step": 9
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.002,
+      "grad_norm": 0.4813508093357086,
+      "kl": 0.0007482333021471277,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 0.0,
+      "num_tokens": 86648.0,
+      "reward": 0.678955078125,
+      "reward_std": 0.012709339149296284,
+      "rewards//mean": 0.678955078125,
+      "rewards//std": 0.05669618770480156,
+      "step": 10
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0022,
+      "grad_norm": 0.47774913907051086,
+      "kl": 0.0006810418926761486,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.0,
+      "num_tokens": 95456.0,
+      "reward": 0.70257568359375,
+      "reward_std": 0.016511568799614906,
+      "rewards//mean": 0.70257568359375,
+      "rewards//std": 0.05309594050049782,
+      "step": 11
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0024,
+      "grad_norm": 0.45333075523376465,
+      "kl": 0.0007765620030113496,
+      "learning_rate": 1.1e-06,
+      "loss": 0.0,
+      "num_tokens": 104112.0,
+      "reward": 0.701904296875,
+      "reward_std": 0.014031937345862389,
+      "rewards//mean": 0.701904296875,
+      "rewards//std": 0.04883308708667755,
+      "step": 12
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0026,
+      "grad_norm": 0.4627256989479065,
+      "kl": 0.0007952243977342732,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 0.0,
+      "num_tokens": 112728.0,
+      "reward": 0.7293701171875,
+      "reward_std": 0.013355368748307228,
+      "rewards//mean": 0.7293701171875,
+      "rewards//std": 0.048412635922431946,
+      "step": 13
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0028,
+      "grad_norm": 0.529376745223999,
+      "kl": 0.0007697402106714435,
+      "learning_rate": 1.3e-06,
+      "loss": 0.0,
+      "num_tokens": 121352.0,
+      "reward": 0.75030517578125,
+      "reward_std": 0.017826953902840614,
+      "rewards//mean": 0.75030517578125,
+      "rewards//std": 0.044097088277339935,
+      "step": 14
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.003,
+      "grad_norm": 0.5399572253227234,
+      "kl": 0.0007980391455930658,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 0.0,
+      "num_tokens": 130072.0,
+      "reward": 0.7301025390625,
+      "reward_std": 0.016819626092910767,
+      "rewards//mean": 0.7301025390625,
+      "rewards//std": 0.06331897526979446,
+      "step": 15
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0032,
+      "grad_norm": 0.48803454637527466,
+      "kl": 0.0007127898643375374,
+      "learning_rate": 1.5e-06,
+      "loss": 0.0,
+      "num_tokens": 138680.0,
+      "reward": 0.72723388671875,
+      "reward_std": 0.01550104096531868,
+      "rewards//mean": 0.72723388671875,
+      "rewards//std": 0.05276963487267494,
+      "step": 16
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0034,
+      "grad_norm": 0.4796734154224396,
+      "kl": 0.0007568690125481226,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.0,
+      "num_tokens": 147320.0,
+      "reward": 0.67901611328125,
+      "reward_std": 0.012344243004918098,
+      "rewards//mean": 0.67901611328125,
+      "rewards//std": 0.05607627332210541,
+      "step": 17
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0036,
+      "grad_norm": 0.4370866119861603,
+      "kl": 0.0007120481022866443,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 0.0,
+      "num_tokens": 155984.0,
+      "reward": 0.7203369140625,
+      "reward_std": 0.010229130275547504,
+      "rewards//mean": 0.7203369140625,
+      "rewards//std": 0.04977063462138176,
+      "step": 18
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0038,
+      "grad_norm": 0.47938504815101624,
+      "kl": 0.0007277187978615984,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 0.0,
+      "num_tokens": 164608.0,
+      "reward": 0.702392578125,
+      "reward_std": 0.020038627088069916,
+      "rewards//mean": 0.702392578125,
+      "rewards//std": 0.06601254642009735,
+      "step": 19
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.004,
+      "grad_norm": 0.5836920142173767,
+      "kl": 0.000848565723572392,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 0.0,
+      "num_tokens": 173168.0,
+      "reward": 0.71051025390625,
+      "reward_std": 0.016375649720430374,
+      "rewards//mean": 0.71051025390625,
+      "rewards//std": 0.0618068166077137,
+      "step": 20
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0042,
+      "grad_norm": 0.4508132040500641,
+      "kl": 0.0007047736216918565,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0,
+      "num_tokens": 181792.0,
+      "reward": 0.71771240234375,
+      "reward_std": 0.01673172414302826,
+      "rewards//mean": 0.71771240234375,
+      "rewards//std": 0.06672913581132889,
+      "step": 21
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0044,
+      "grad_norm": 0.4673251509666443,
+      "kl": 0.0007006596788414754,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 0.0,
+      "num_tokens": 190392.0,
+      "reward": 0.72503662109375,
+      "reward_std": 0.016404539346694946,
+      "rewards//mean": 0.72503662109375,
+      "rewards//std": 0.04331747442483902,
+      "step": 22
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0046,
+      "grad_norm": 0.752646267414093,
+      "kl": 0.0010592954204184934,
+      "learning_rate": 2.2e-06,
+      "loss": 0.0,
+      "num_tokens": 199080.0,
+      "reward": 0.70477294921875,
+      "reward_std": 0.013751041144132614,
+      "rewards//mean": 0.70477294921875,
+      "rewards//std": 0.05169031769037247,
+      "step": 23
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0048,
+      "grad_norm": 0.5306899547576904,
+      "kl": 0.000849992771691177,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 0.0,
+      "num_tokens": 207720.0,
+      "reward": 0.7218017578125,
+      "reward_std": 0.017207415774464607,
+      "rewards//mean": 0.7218017578125,
+      "rewards//std": 0.0644102469086647,
+      "step": 24
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.005,
+      "grad_norm": 0.44910290837287903,
+      "kl": 0.0007659951588721015,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 0.0,
+      "num_tokens": 216296.0,
+      "reward": 0.70526123046875,
+      "reward_std": 0.01439366303384304,
+      "rewards//mean": 0.70526123046875,
+      "rewards//std": 0.060455020517110825,
+      "step": 25
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0052,
+      "grad_norm": 0.5068986415863037,
+      "kl": 0.0008941922133089975,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0,
+      "num_tokens": 224968.0,
+      "reward": 0.732421875,
+      "reward_std": 0.017151644453406334,
+      "rewards//mean": 0.732421875,
+      "rewards//std": 0.05576201528310776,
+      "step": 26
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0054,
+      "grad_norm": 0.5920788645744324,
+      "kl": 0.0007423345305141993,
+      "learning_rate": 2.6e-06,
+      "loss": 0.0,
+      "num_tokens": 233528.0,
+      "reward": 0.6807861328125,
+      "reward_std": 0.018578065559267998,
+      "rewards//mean": 0.6807861328125,
+      "rewards//std": 0.0869307816028595,
+      "step": 27
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0056,
+      "grad_norm": 0.5785125494003296,
+      "kl": 0.0010453350987518206,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 0.0,
+      "num_tokens": 242280.0,
+      "reward": 0.68853759765625,
+      "reward_std": 0.016351919621229172,
+      "rewards//mean": 0.68853759765625,
+      "rewards//std": 0.06393762677907944,
+      "step": 28
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0058,
+      "grad_norm": 0.503767728805542,
+      "kl": 0.0008367645714315586,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 0.0,
+      "num_tokens": 250976.0,
+      "reward": 0.71185302734375,
+      "reward_std": 0.01879560574889183,
+      "rewards//mean": 0.71185302734375,
+      "rewards//std": 0.05082531273365021,
+      "step": 29
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.006,
+      "grad_norm": 0.5023078918457031,
+      "kl": 0.0007173106641857885,
+      "learning_rate": 2.9e-06,
+      "loss": 0.0,
+      "num_tokens": 259632.0,
+      "reward": 0.70556640625,
+      "reward_std": 0.017621025443077087,
+      "rewards//mean": 0.70556640625,
+      "rewards//std": 0.0475139394402504,
+      "step": 30
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0062,
+      "grad_norm": 0.4851153492927551,
+      "kl": 0.0007923798766569234,
+      "learning_rate": 3e-06,
+      "loss": 0.0,
+      "num_tokens": 268208.0,
+      "reward": 0.7398681640625,
+      "reward_std": 0.01781066693365574,
+      "rewards//mean": 0.7398681640625,
+      "rewards//std": 0.05101071298122406,
+      "step": 31
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0064,
+      "grad_norm": 0.4202479422092438,
+      "kl": 0.0007000941404839978,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 0.0,
+      "num_tokens": 276824.0,
+      "reward": 0.711669921875,
+      "reward_std": 0.014990320429205894,
+      "rewards//mean": 0.711669921875,
+      "rewards//std": 0.05982654541730881,
+      "step": 32
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0066,
+      "grad_norm": 0.5245676636695862,
+      "kl": 0.0007394420899800025,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 0.0,
+      "num_tokens": 285520.0,
+      "reward": 0.73822021484375,
+      "reward_std": 0.014528016559779644,
+      "rewards//mean": 0.73822021484375,
+      "rewards//std": 0.05750894173979759,
+      "step": 33
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0068,
+      "grad_norm": 0.41506427526474,
+      "kl": 0.0006638994673267007,
+      "learning_rate": 3.3000000000000006e-06,
+      "loss": 0.0,
+      "num_tokens": 294144.0,
+      "reward": 0.705078125,
+      "reward_std": 0.018270526081323624,
+      "rewards//mean": 0.705078125,
+      "rewards//std": 0.05205077305436134,
+      "step": 34
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.007,
+      "grad_norm": 0.6635000705718994,
+      "kl": 0.0008408781359321438,
+      "learning_rate": 3.4000000000000005e-06,
+      "loss": 0.0,
+      "num_tokens": 302888.0,
+      "reward": 0.7369384765625,
+      "reward_std": 0.010665340349078178,
+      "rewards//mean": 0.7369384765625,
+      "rewards//std": 0.041731830686330795,
+      "step": 35
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0072,
+      "grad_norm": 0.5000082850456238,
+      "kl": 0.0007324635953409597,
+      "learning_rate": 3.5e-06,
+      "loss": 0.0,
+      "num_tokens": 311504.0,
+      "reward": 0.7015380859375,
+      "reward_std": 0.015993408858776093,
+      "rewards//mean": 0.7015380859375,
+      "rewards//std": 0.05921553075313568,
+      "step": 36
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0074,
+      "grad_norm": 0.4788602292537689,
+      "kl": 0.0007234790246002376,
+      "learning_rate": 3.6000000000000003e-06,
+      "loss": 0.0,
+      "num_tokens": 320208.0,
+      "reward": 0.71978759765625,
+      "reward_std": 0.015359117649495602,
+      "rewards//mean": 0.71978759765625,
+      "rewards//std": 0.050679761916399,
+      "step": 37
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0076,
+      "grad_norm": 0.5380129218101501,
+      "kl": 0.0006790131737943739,
+      "learning_rate": 3.7e-06,
+      "loss": 0.0,
+      "num_tokens": 328904.0,
+      "reward": 0.71307373046875,
+      "reward_std": 0.017655834555625916,
+      "rewards//mean": 0.71307373046875,
+      "rewards//std": 0.039130210876464844,
+      "step": 38
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0078,
+      "grad_norm": 0.47228410840034485,
+      "kl": 0.0006949657399673015,
+      "learning_rate": 3.8000000000000005e-06,
+      "loss": 0.0,
+      "num_tokens": 337632.0,
+      "reward": 0.705322265625,
+      "reward_std": 0.01650204323232174,
+      "rewards//mean": 0.705322265625,
+      "rewards//std": 0.0695849284529686,
+      "step": 39
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.008,
+      "grad_norm": 0.5206338763237,
+      "kl": 0.0007481320935767144,
+      "learning_rate": 3.900000000000001e-06,
+      "loss": 0.0,
+      "num_tokens": 346264.0,
+      "reward": 0.69219970703125,
+      "reward_std": 0.016963649541139603,
+      "rewards//mean": 0.69219970703125,
+      "rewards//std": 0.05523912236094475,
+      "step": 40
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0082,
+      "grad_norm": 0.49259892106056213,
+      "kl": 0.0007038385519990698,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.0,
+      "num_tokens": 354912.0,
+      "reward": 0.71923828125,
+      "reward_std": 0.018238678574562073,
+      "rewards//mean": 0.71923828125,
+      "rewards//std": 0.060513366013765335,
+      "step": 41
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0084,
+      "grad_norm": 0.42862868309020996,
+      "kl": 0.0006526454817503691,
+      "learning_rate": 4.1e-06,
+      "loss": 0.0,
+      "num_tokens": 363544.0,
+      "reward": 0.73712158203125,
+      "reward_std": 0.012668399140238762,
+      "rewards//mean": 0.73712158203125,
+      "rewards//std": 0.05271969735622406,
+      "step": 42
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0086,
+      "grad_norm": 0.43543925881385803,
+      "kl": 0.0007449448385159485,
+      "learning_rate": 4.2000000000000004e-06,
+      "loss": 0.0,
+      "num_tokens": 372144.0,
+      "reward": 0.73724365234375,
+      "reward_std": 0.017016027122735977,
+      "rewards//mean": 0.73724365234375,
+      "rewards//std": 0.04933328554034233,
+      "step": 43
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0088,
+      "grad_norm": 0.5209327340126038,
+      "kl": 0.0007311399604077451,
+      "learning_rate": 4.3e-06,
+      "loss": 0.0,
+      "num_tokens": 380848.0,
+      "reward": 0.7359619140625,
+      "reward_std": 0.010918927378952503,
+      "rewards//mean": 0.7359619140625,
+      "rewards//std": 0.0476677305996418,
+      "step": 44
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.009,
+      "grad_norm": 0.5357726216316223,
+      "kl": 0.0008118156329146586,
+      "learning_rate": 4.4e-06,
+      "loss": 0.0,
+      "num_tokens": 389504.0,
+      "reward": 0.66937255859375,
+      "reward_std": 0.01259728241711855,
+      "rewards//mean": 0.66937255859375,
+      "rewards//std": 0.051660146564245224,
+      "step": 45
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0092,
+      "grad_norm": 0.45110824704170227,
+      "kl": 0.0006952350158826448,
+      "learning_rate": 4.5e-06,
+      "loss": 0.0,
+      "num_tokens": 398200.0,
+      "reward": 0.724853515625,
+      "reward_std": 0.013849527575075626,
+      "rewards//mean": 0.724853515625,
+      "rewards//std": 0.0702190026640892,
+      "step": 46
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0094,
+      "grad_norm": 0.47275251150131226,
+      "kl": 0.0007834887073840946,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 0.0,
+      "num_tokens": 406848.0,
+      "reward": 0.69769287109375,
+      "reward_std": 0.015200081281363964,
+      "rewards//mean": 0.69769287109375,
+      "rewards//std": 0.07077126950025558,
+      "step": 47
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0096,
+      "grad_norm": 0.42265617847442627,
+      "kl": 0.000766771991038695,
+      "learning_rate": 4.7e-06,
+      "loss": 0.0,
+      "num_tokens": 415520.0,
+      "reward": 0.70733642578125,
+      "reward_std": 0.017255615442991257,
+      "rewards//mean": 0.70733642578125,
+      "rewards//std": 0.0646086260676384,
+      "step": 48
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0098,
+      "grad_norm": 0.46693459153175354,
+      "kl": 0.0007007038075244054,
+      "learning_rate": 4.800000000000001e-06,
+      "loss": 0.0,
+      "num_tokens": 424088.0,
+      "reward": 0.7169189453125,
+      "reward_std": 0.017792128026485443,
+      "rewards//mean": 0.7169189453125,
+      "rewards//std": 0.06479453295469284,
+      "step": 49
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.01,
+      "grad_norm": 0.46468374133110046,
+      "kl": 0.0007802178952260874,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.0,
+      "num_tokens": 432696.0,
+      "reward": 0.707275390625,
+      "reward_std": 0.01039247214794159,
+      "rewards//mean": 0.707275390625,
+      "rewards//std": 0.06051086261868477,
+      "step": 50
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0102,
+      "grad_norm": 0.4285586476325989,
+      "kl": 0.000754905202484224,
+      "learning_rate": 5e-06,
+      "loss": 0.0,
+      "num_tokens": 441352.0,
+      "reward": 0.72601318359375,
+      "reward_std": 0.015443078242242336,
+      "rewards//mean": 0.72601318359375,
+      "rewards//std": 0.04845963418483734,
+      "step": 51
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0104,
+      "grad_norm": 0.46686625480651855,
+      "kl": 0.00072819792694645,
+      "learning_rate": 4.9999994965001495e-06,
+      "loss": 0.0,
+      "num_tokens": 449960.0,
+      "reward": 0.7313232421875,
+      "reward_std": 0.013708039186894894,
+      "rewards//mean": 0.7313232421875,
+      "rewards//std": 0.057744208723306656,
+      "step": 52
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0106,
+      "grad_norm": 0.5042674541473389,
+      "kl": 0.0007860148616600782,
+      "learning_rate": 4.999997986000801e-06,
+      "loss": 0.0,
+      "num_tokens": 458512.0,
+      "reward": 0.71722412109375,
+      "reward_std": 0.015048246830701828,
+      "rewards//mean": 0.71722412109375,
+      "rewards//std": 0.05379721522331238,
+      "step": 53
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0108,
+      "grad_norm": 0.4119553864002228,
+      "kl": 0.0006910300071467645,
+      "learning_rate": 4.999995468502563e-06,
+      "loss": 0.0,
+      "num_tokens": 467048.0,
+      "reward": 0.73016357421875,
+      "reward_std": 0.01429541502147913,
+      "rewards//mean": 0.73016357421875,
+      "rewards//std": 0.054377954453229904,
+      "step": 54
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.011,
+      "grad_norm": 0.4966086745262146,
+      "kl": 0.0007905790334916674,
+      "learning_rate": 4.9999919440064484e-06,
+      "loss": 0.0,
+      "num_tokens": 475728.0,
+      "reward": 0.69805908203125,
+      "reward_std": 0.016646619886159897,
+      "rewards//mean": 0.69805908203125,
+      "rewards//std": 0.06764635443687439,
+      "step": 55
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0112,
+      "grad_norm": 0.4910055100917816,
+      "kl": 0.0007386217839666642,
+      "learning_rate": 4.999987412513878e-06,
+      "loss": 0.0,
+      "num_tokens": 484360.0,
+      "reward": 0.72802734375,
+      "reward_std": 0.015091042965650558,
+      "rewards//mean": 0.72802734375,
+      "rewards//std": 0.06086854264140129,
+      "step": 56
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0114,
+      "grad_norm": 0.4628139138221741,
+      "kl": 0.0007929667408461682,
+      "learning_rate": 4.999981874026677e-06,
+      "loss": 0.0,
+      "num_tokens": 493000.0,
+      "reward": 0.72283935546875,
+      "reward_std": 0.015091566368937492,
+      "rewards//mean": 0.72283935546875,
+      "rewards//std": 0.056771133095026016,
+      "step": 57
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0116,
+      "grad_norm": 0.5012953877449036,
+      "kl": 0.0007925045429146849,
+      "learning_rate": 4.9999753285470756e-06,
+      "loss": 0.0,
+      "num_tokens": 501632.0,
+      "reward": 0.71661376953125,
+      "reward_std": 0.015197820030152798,
+      "rewards//mean": 0.71661376953125,
+      "rewards//std": 0.051168158650398254,
+      "step": 58
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0118,
+      "grad_norm": 0.5567752718925476,
+      "kl": 0.0009017333650263026,
+      "learning_rate": 4.9999677760777114e-06,
+      "loss": 0.0,
+      "num_tokens": 510288.0,
+      "reward": 0.72381591796875,
+      "reward_std": 0.013100363314151764,
+      "rewards//mean": 0.72381591796875,
+      "rewards//std": 0.04677506536245346,
+      "step": 59
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.012,
+      "grad_norm": 0.41019734740257263,
+      "kl": 0.0007576563366455957,
+      "learning_rate": 4.999959216621626e-06,
+      "loss": 0.0,
+      "num_tokens": 518952.0,
+      "reward": 0.721435546875,
+      "reward_std": 0.01749395951628685,
+      "rewards//mean": 0.721435546875,
+      "rewards//std": 0.0656224712729454,
+      "step": 60
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0122,
+      "grad_norm": 0.5433821082115173,
+      "kl": 0.0012775656068697572,
+      "learning_rate": 4.999949650182267e-06,
+      "loss": 0.0001,
+      "num_tokens": 527520.0,
+      "reward": 0.71600341796875,
+      "reward_std": 0.014195160940289497,
+      "rewards//mean": 0.71600341796875,
+      "rewards//std": 0.040854718536138535,
+      "step": 61
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0124,
+      "grad_norm": 0.4710741639137268,
+      "kl": 0.0007979272995726205,
+      "learning_rate": 4.999939076763487e-06,
+      "loss": 0.0,
+      "num_tokens": 536112.0,
+      "reward": 0.7099609375,
+      "reward_std": 0.015660030767321587,
+      "rewards//mean": 0.7099609375,
+      "rewards//std": 0.0644836500287056,
+      "step": 62
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0126,
+      "grad_norm": 0.5351571440696716,
+      "kl": 0.0008906932343961671,
+      "learning_rate": 4.999927496369547e-06,
+      "loss": 0.0,
+      "num_tokens": 544736.0,
+      "reward": 0.6954345703125,
+      "reward_std": 0.01476267259567976,
+      "rewards//mean": 0.6954345703125,
+      "rewards//std": 0.04151507467031479,
+      "step": 63
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0128,
+      "grad_norm": 0.4595547318458557,
+      "kl": 0.0008543600051780231,
+      "learning_rate": 4.99991490900511e-06,
+      "loss": 0.0,
+      "num_tokens": 553408.0,
+      "reward": 0.71673583984375,
+      "reward_std": 0.012672960758209229,
+      "rewards//mean": 0.71673583984375,
+      "rewards//std": 0.05019837245345116,
+      "step": 64
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.013,
+      "grad_norm": 0.4643782079219818,
+      "kl": 0.0008414792246185243,
+      "learning_rate": 4.999901314675246e-06,
+      "loss": 0.0,
+      "num_tokens": 562064.0,
+      "reward": 0.71978759765625,
+      "reward_std": 0.01744549721479416,
+      "rewards//mean": 0.71978759765625,
+      "rewards//std": 0.05583685636520386,
+      "step": 65
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0132,
+      "grad_norm": 0.4610508680343628,
+      "kl": 0.0008548630212317221,
+      "learning_rate": 4.999886713385432e-06,
+      "loss": 0.0,
+      "num_tokens": 570664.0,
+      "reward": 0.71099853515625,
+      "reward_std": 0.016434110701084137,
+      "rewards//mean": 0.71099853515625,
+      "rewards//std": 0.06650462746620178,
+      "step": 66
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0134,
+      "grad_norm": 0.48795291781425476,
+      "kl": 0.000795883170212619,
+      "learning_rate": 4.999871105141549e-06,
+      "loss": 0.0,
+      "num_tokens": 579360.0,
+      "reward": 0.7103271484375,
+      "reward_std": 0.016300387680530548,
+      "rewards//mean": 0.7103271484375,
+      "rewards//std": 0.0523579940199852,
+      "step": 67
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0136,
+      "grad_norm": 0.4637707769870758,
+      "kl": 0.0008550576094421558,
+      "learning_rate": 4.9998544899498845e-06,
+      "loss": 0.0,
+      "num_tokens": 587992.0,
+      "reward": 0.7423095703125,
+      "reward_std": 0.014237109571695328,
+      "rewards//mean": 0.7423095703125,
+      "rewards//std": 0.04064841568470001,
+      "step": 68
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0138,
+      "grad_norm": 0.47836223244667053,
+      "kl": 0.0008403940519201569,
+      "learning_rate": 4.999836867817129e-06,
+      "loss": 0.0,
+      "num_tokens": 596608.0,
+      "reward": 0.69384765625,
+      "reward_std": 0.014926273375749588,
+      "rewards//mean": 0.69384765625,
+      "rewards//std": 0.040478430688381195,
+      "step": 69
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.014,
+      "grad_norm": 0.49338188767433167,
+      "kl": 0.0009159344772342592,
+      "learning_rate": 4.9998182387503825e-06,
+      "loss": 0.0,
+      "num_tokens": 605248.0,
+      "reward": 0.70245361328125,
+      "reward_std": 0.023733031004667282,
+      "rewards//mean": 0.70245361328125,
+      "rewards//std": 0.05866420641541481,
+      "step": 70
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0142,
+      "grad_norm": 0.3988666534423828,
+      "kl": 0.0008121578212012537,
+      "learning_rate": 4.999798602757149e-06,
+      "loss": 0.0,
+      "num_tokens": 613824.0,
+      "reward": 0.69830322265625,
+      "reward_std": 0.013719111680984497,
+      "rewards//mean": 0.69830322265625,
+      "rewards//std": 0.056460171937942505,
+      "step": 71
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0144,
+      "grad_norm": 0.4849626421928406,
+      "kl": 0.0009415504609933123,
+      "learning_rate": 4.9997779598453365e-06,
+      "loss": 0.0,
+      "num_tokens": 622328.0,
+      "reward": 0.73828125,
+      "reward_std": 0.01322929933667183,
+      "rewards//mean": 0.73828125,
+      "rewards//std": 0.06599832326173782,
+      "step": 72
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0146,
+      "grad_norm": 0.5434121489524841,
+      "kl": 0.0016273690489470027,
+      "learning_rate": 4.999756310023261e-06,
+      "loss": 0.0001,
+      "num_tokens": 630976.0,
+      "reward": 0.6895751953125,
+      "reward_std": 0.01436593197286129,
+      "rewards//mean": 0.6895751953125,
+      "rewards//std": 0.04783637821674347,
+      "step": 73
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0148,
+      "grad_norm": 0.4288107454776764,
+      "kl": 0.0008081157502601855,
+      "learning_rate": 4.999733653299643e-06,
+      "loss": 0.0,
+      "num_tokens": 639624.0,
+      "reward": 0.73236083984375,
+      "reward_std": 0.015878882259130478,
+      "rewards//mean": 0.73236083984375,
+      "rewards//std": 0.040517259389162064,
+      "step": 74
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.015,
+      "grad_norm": 0.48302745819091797,
+      "kl": 0.0009941107928170823,
+      "learning_rate": 4.9997099896836076e-06,
+      "loss": 0.0,
+      "num_tokens": 648320.0,
+      "reward": 0.69451904296875,
+      "reward_std": 0.013520974665880203,
+      "rewards//mean": 0.69451904296875,
+      "rewards//std": 0.06670349836349487,
+      "step": 75
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0152,
+      "grad_norm": 0.4562317728996277,
+      "kl": 0.0008667359288665466,
+      "learning_rate": 4.999685319184688e-06,
+      "loss": 0.0,
+      "num_tokens": 657080.0,
+      "reward": 0.7342529296875,
+      "reward_std": 0.013470092788338661,
+      "rewards//mean": 0.7342529296875,
+      "rewards//std": 0.047755297273397446,
+      "step": 76
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0154,
+      "grad_norm": 0.491922527551651,
+      "kl": 0.000954871931753587,
+      "learning_rate": 4.999659641812821e-06,
+      "loss": 0.0,
+      "num_tokens": 665720.0,
+      "reward": 0.70245361328125,
+      "reward_std": 0.015949761494994164,
+      "rewards//mean": 0.70245361328125,
+      "rewards//std": 0.07394102215766907,
+      "step": 77
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0156,
+      "grad_norm": 0.4906434118747711,
+      "kl": 0.0010538664282648824,
+      "learning_rate": 4.9996329575783486e-06,
+      "loss": 0.0,
+      "num_tokens": 674336.0,
+      "reward": 0.71417236328125,
+      "reward_std": 0.016342610120773315,
+      "rewards//mean": 0.71417236328125,
+      "rewards//std": 0.05462348833680153,
+      "step": 78
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0158,
+      "grad_norm": 0.488397479057312,
+      "kl": 0.0009591389462002553,
+      "learning_rate": 4.99960526649202e-06,
+      "loss": 0.0,
+      "num_tokens": 682968.0,
+      "reward": 0.68719482421875,
+      "reward_std": 0.01782170683145523,
+      "rewards//mean": 0.68719482421875,
+      "rewards//std": 0.07477407157421112,
+      "step": 79
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.016,
+      "grad_norm": 0.4663703143596649,
+      "kl": 0.0009951836473192088,
+      "learning_rate": 4.999576568564989e-06,
+      "loss": 0.0,
+      "num_tokens": 691640.0,
+      "reward": 0.73382568359375,
+      "reward_std": 0.011371012777090073,
+      "rewards//mean": 0.73382568359375,
+      "rewards//std": 0.06883952021598816,
+      "step": 80
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0162,
+      "grad_norm": 0.43239349126815796,
+      "kl": 0.0008809622740955092,
+      "learning_rate": 4.999546863808815e-06,
+      "loss": 0.0,
+      "num_tokens": 700264.0,
+      "reward": 0.6995849609375,
+      "reward_std": 0.011680185794830322,
+      "rewards//mean": 0.6995849609375,
+      "rewards//std": 0.04613397642970085,
+      "step": 81
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0164,
+      "grad_norm": 0.446817010641098,
+      "kl": 0.0009117565277847461,
+      "learning_rate": 4.999516152235463e-06,
+      "loss": 0.0,
+      "num_tokens": 708984.0,
+      "reward": 0.73577880859375,
+      "reward_std": 0.010488376021385193,
+      "rewards//mean": 0.73577880859375,
+      "rewards//std": 0.05107547715306282,
+      "step": 82
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0166,
+      "grad_norm": 0.43615901470184326,
+      "kl": 0.0009348735839012079,
+      "learning_rate": 4.999484433857305e-06,
+      "loss": 0.0,
+      "num_tokens": 717568.0,
+      "reward": 0.723388671875,
+      "reward_std": 0.013231704942882061,
+      "rewards//mean": 0.723388671875,
+      "rewards//std": 0.0505966916680336,
+      "step": 83
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0168,
+      "grad_norm": 0.496662974357605,
+      "kl": 0.0011133336302009411,
+      "learning_rate": 4.999451708687114e-06,
+      "loss": 0.0,
+      "num_tokens": 726304.0,
+      "reward": 0.7135009765625,
+      "reward_std": 0.017629370093345642,
+      "rewards//mean": 0.7135009765625,
+      "rewards//std": 0.05801933631300926,
+      "step": 84
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.017,
+      "grad_norm": 0.5092006921768188,
+      "kl": 0.0010797181967063807,
+      "learning_rate": 4.999417976738075e-06,
+      "loss": 0.0,
+      "num_tokens": 735000.0,
+      "reward": 0.72796630859375,
+      "reward_std": 0.011926619336009026,
+      "rewards//mean": 0.72796630859375,
+      "rewards//std": 0.06286849081516266,
+      "step": 85
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0172,
+      "grad_norm": 0.4398786425590515,
+      "kl": 0.0010526972182560712,
+      "learning_rate": 4.999383238023773e-06,
+      "loss": 0.0,
+      "num_tokens": 743648.0,
+      "reward": 0.70025634765625,
+      "reward_std": 0.013448704965412617,
+      "rewards//mean": 0.70025634765625,
+      "rewards//std": 0.07408399134874344,
+      "step": 86
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0174,
+      "grad_norm": 0.5147225260734558,
+      "kl": 0.0012325849456829019,
+      "learning_rate": 4.999347492558203e-06,
+      "loss": 0.0,
+      "num_tokens": 752416.0,
+      "reward": 0.71722412109375,
+      "reward_std": 0.014034003019332886,
+      "rewards//mean": 0.71722412109375,
+      "rewards//std": 0.07134390622377396,
+      "step": 87
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0176,
+      "grad_norm": 0.5070951581001282,
+      "kl": 0.001082870177924633,
+      "learning_rate": 4.999310740355761e-06,
+      "loss": 0.0,
+      "num_tokens": 761040.0,
+      "reward": 0.7266845703125,
+      "reward_std": 0.014604616910219193,
+      "rewards//mean": 0.7266845703125,
+      "rewards//std": 0.06855076551437378,
+      "step": 88
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0178,
+      "grad_norm": 0.4914563298225403,
+      "kl": 0.0012297793437028304,
+      "learning_rate": 4.9992729814312514e-06,
+      "loss": 0.0,
+      "num_tokens": 769656.0,
+      "reward": 0.74017333984375,
+      "reward_std": 0.019316695630550385,
+      "rewards//mean": 0.74017333984375,
+      "rewards//std": 0.06556382775306702,
+      "step": 89
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.018,
+      "grad_norm": 0.4564233124256134,
+      "kl": 0.0011137609835714102,
+      "learning_rate": 4.999234215799884e-06,
+      "loss": 0.0,
+      "num_tokens": 778248.0,
+      "reward": 0.697509765625,
+      "reward_std": 0.015206321142613888,
+      "rewards//mean": 0.697509765625,
+      "rewards//std": 0.019185619428753853,
+      "step": 90
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0182,
+      "grad_norm": 0.5231274366378784,
+      "kl": 0.0012513935289462097,
+      "learning_rate": 4.999194443477273e-06,
+      "loss": 0.0001,
+      "num_tokens": 786856.0,
+      "reward": 0.689697265625,
+      "reward_std": 0.018568219617009163,
+      "rewards//mean": 0.689697265625,
+      "rewards//std": 0.08479659259319305,
+      "step": 91
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0184,
+      "grad_norm": 0.45985209941864014,
+      "kl": 0.0012780725373886526,
+      "learning_rate": 4.99915366447944e-06,
+      "loss": 0.0001,
+      "num_tokens": 795544.0,
+      "reward": 0.6876220703125,
+      "reward_std": 0.016050245612859726,
+      "rewards//mean": 0.6876220703125,
+      "rewards//std": 0.06961309164762497,
+      "step": 92
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0186,
+      "grad_norm": 0.4588201940059662,
+      "kl": 0.0010780094598885626,
+      "learning_rate": 4.999111878822809e-06,
+      "loss": 0.0,
+      "num_tokens": 804104.0,
+      "reward": 0.73370361328125,
+      "reward_std": 0.015261776745319366,
+      "rewards//mean": 0.73370361328125,
+      "rewards//std": 0.05237804725766182,
+      "step": 93
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0188,
+      "grad_norm": 0.4953310489654541,
+      "kl": 0.0013287549882079475,
+      "learning_rate": 4.999069086524212e-06,
+      "loss": 0.0001,
+      "num_tokens": 812768.0,
+      "reward": 0.74371337890625,
+      "reward_std": 0.017101481556892395,
+      "rewards//mean": 0.74371337890625,
+      "rewards//std": 0.07072333991527557,
+      "step": 94
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.019,
+      "grad_norm": 0.5649318099021912,
+      "kl": 0.0012824844452552497,
+      "learning_rate": 4.999025287600886e-06,
+      "loss": 0.0001,
+      "num_tokens": 821400.0,
+      "reward": 0.73883056640625,
+      "reward_std": 0.012025142088532448,
+      "rewards//mean": 0.73883056640625,
+      "rewards//std": 0.04969830811023712,
+      "step": 95
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0192,
+      "grad_norm": 0.6112560629844666,
+      "kl": 0.0013404397759586573,
+      "learning_rate": 4.998980482070473e-06,
+      "loss": 0.0001,
+      "num_tokens": 829952.0,
+      "reward": 0.68670654296875,
+      "reward_std": 0.015257102437317371,
+      "rewards//mean": 0.68670654296875,
+      "rewards//std": 0.03606545925140381,
+      "step": 96
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0194,
+      "grad_norm": 0.5092772841453552,
+      "kl": 0.0011790767530328594,
+      "learning_rate": 4.9989346699510215e-06,
+      "loss": 0.0,
+      "num_tokens": 838520.0,
+      "reward": 0.71392822265625,
+      "reward_std": 0.013484635390341282,
+      "rewards//mean": 0.71392822265625,
+      "rewards//std": 0.04560608044266701,
+      "step": 97
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0196,
+      "grad_norm": 0.4867624044418335,
+      "kl": 0.001275954389711842,
+      "learning_rate": 4.9988878512609825e-06,
+      "loss": 0.0001,
+      "num_tokens": 847128.0,
+      "reward": 0.708740234375,
+      "reward_std": 0.014133438467979431,
+      "rewards//mean": 0.708740234375,
+      "rewards//std": 0.0834752693772316,
+      "step": 98
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0198,
+      "grad_norm": 0.5237188935279846,
+      "kl": 0.001374387225951068,
+      "learning_rate": 4.998840026019217e-06,
+      "loss": 0.0001,
+      "num_tokens": 855824.0,
+      "reward": 0.6898193359375,
+      "reward_std": 0.014914432540535927,
+      "rewards//mean": 0.6898193359375,
+      "rewards//std": 0.05682779848575592,
+      "step": 99
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.02,
+      "grad_norm": 0.4966977834701538,
+      "kl": 0.0014171117945807055,
+      "learning_rate": 4.998791194244988e-06,
+      "loss": 0.0001,
+      "num_tokens": 864520.0,
+      "reward": 0.71148681640625,
+      "reward_std": 0.014294968917965889,
+      "rewards//mean": 0.71148681640625,
+      "rewards//std": 0.08842907845973969,
+      "step": 100
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0202,
+      "grad_norm": 0.9231608510017395,
+      "kl": 0.0014634677936555818,
+      "learning_rate": 4.998741355957963e-06,
+      "loss": 0.0001,
+      "num_tokens": 873152.0,
+      "reward": 0.709228515625,
+      "reward_std": 0.015058237127959728,
+      "rewards//mean": 0.709228515625,
+      "rewards//std": 0.07786376029253006,
+      "step": 101
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0204,
+      "grad_norm": 0.49847280979156494,
+      "kl": 0.001440991909475997,
+      "learning_rate": 4.99869051117822e-06,
+      "loss": 0.0001,
+      "num_tokens": 881824.0,
+      "reward": 0.7056884765625,
+      "reward_std": 0.014790365472435951,
+      "rewards//mean": 0.7056884765625,
+      "rewards//std": 0.05627746134996414,
+      "step": 102
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0206,
+      "grad_norm": 0.5402097702026367,
+      "kl": 0.0013479594781529158,
+      "learning_rate": 4.998638659926238e-06,
+      "loss": 0.0001,
+      "num_tokens": 890368.0,
+      "reward": 0.74066162109375,
+      "reward_std": 0.014553939923644066,
+      "rewards//mean": 0.74066162109375,
+      "rewards//std": 0.05418301746249199,
+      "step": 103
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0208,
+      "grad_norm": 0.5026318430900574,
+      "kl": 0.001596087298821658,
+      "learning_rate": 4.998585802222902e-06,
+      "loss": 0.0001,
+      "num_tokens": 899128.0,
+      "reward": 0.74249267578125,
+      "reward_std": 0.022882359102368355,
+      "rewards//mean": 0.74249267578125,
+      "rewards//std": 0.04922054708003998,
+      "step": 104
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.021,
+      "grad_norm": 0.4722844958305359,
+      "kl": 0.0016166606947081164,
+      "learning_rate": 4.9985319380895035e-06,
+      "loss": 0.0001,
+      "num_tokens": 907808.0,
+      "reward": 0.7147216796875,
+      "reward_std": 0.01173408329486847,
+      "rewards//mean": 0.7147216796875,
+      "rewards//std": 0.05263826996088028,
+      "step": 105
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0212,
+      "grad_norm": 0.5120245218276978,
+      "kl": 0.0016622742696199566,
+      "learning_rate": 4.99847706754774e-06,
+      "loss": 0.0001,
+      "num_tokens": 916384.0,
+      "reward": 0.74267578125,
+      "reward_std": 0.013011830858886242,
+      "rewards//mean": 0.74267578125,
+      "rewards//std": 0.04836273938417435,
+      "step": 106
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0214,
+      "grad_norm": 0.5261712670326233,
+      "kl": 0.0015574333810945973,
+      "learning_rate": 4.998421190619712e-06,
+      "loss": 0.0001,
+      "num_tokens": 925000.0,
+      "reward": 0.728759765625,
+      "reward_std": 0.01469873171299696,
+      "rewards//mean": 0.728759765625,
+      "rewards//std": 0.029047802090644836,
+      "step": 107
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0216,
+      "grad_norm": 0.4722696542739868,
+      "kl": 0.0016149339353432879,
+      "learning_rate": 4.998364307327927e-06,
+      "loss": 0.0001,
+      "num_tokens": 933680.0,
+      "reward": 0.72637939453125,
+      "reward_std": 0.014723455533385277,
+      "rewards//mean": 0.72637939453125,
+      "rewards//std": 0.057769205421209335,
+      "step": 108
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0218,
+      "grad_norm": 0.471443235874176,
+      "kl": 0.00147363574069459,
+      "learning_rate": 4.998306417695298e-06,
+      "loss": 0.0001,
+      "num_tokens": 942360.0,
+      "reward": 0.70672607421875,
+      "reward_std": 0.013419017195701599,
+      "rewards//mean": 0.70672607421875,
+      "rewards//std": 0.061058465391397476,
+      "step": 109
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.022,
+      "grad_norm": 0.6062172651290894,
+      "kl": 0.0026222791202599183,
+      "learning_rate": 4.998247521745142e-06,
+      "loss": 0.0001,
+      "num_tokens": 951000.0,
+      "reward": 0.74151611328125,
+      "reward_std": 0.018515393137931824,
+      "rewards//mean": 0.74151611328125,
+      "rewards//std": 0.05745021253824234,
+      "step": 110
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0222,
+      "grad_norm": 0.6000130772590637,
+      "kl": 0.0021851205674465746,
+      "learning_rate": 4.998187619501185e-06,
+      "loss": 0.0001,
+      "num_tokens": 959688.0,
+      "reward": 0.674560546875,
+      "reward_std": 0.01506463810801506,
+      "rewards//mean": 0.674560546875,
+      "rewards//std": 0.07186568528413773,
+      "step": 111
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0224,
+      "grad_norm": 0.4527592062950134,
+      "kl": 0.0016580721858190373,
+      "learning_rate": 4.998126710987552e-06,
+      "loss": 0.0001,
+      "num_tokens": 968352.0,
+      "reward": 0.72381591796875,
+      "reward_std": 0.014463772997260094,
+      "rewards//mean": 0.72381591796875,
+      "rewards//std": 0.05746760219335556,
+      "step": 112
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0226,
+      "grad_norm": 0.5143523812294006,
+      "kl": 0.002175911722588353,
+      "learning_rate": 4.998064796228779e-06,
+      "loss": 0.0001,
+      "num_tokens": 976936.0,
+      "reward": 0.7242431640625,
+      "reward_std": 0.014481933787465096,
+      "rewards//mean": 0.7242431640625,
+      "rewards//std": 0.05567820370197296,
+      "step": 113
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0228,
+      "grad_norm": 0.48747143149375916,
+      "kl": 0.0017545891605550423,
+      "learning_rate": 4.998001875249804e-06,
+      "loss": 0.0001,
+      "num_tokens": 985552.0,
+      "reward": 0.71710205078125,
+      "reward_std": 0.013459138572216034,
+      "rewards//mean": 0.71710205078125,
+      "rewards//std": 0.04908997192978859,
+      "step": 114
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.023,
+      "grad_norm": 0.4331216514110565,
+      "kl": 0.0021779875096399337,
+      "learning_rate": 4.997937948075973e-06,
+      "loss": 0.0001,
+      "num_tokens": 994264.0,
+      "reward": 0.72625732421875,
+      "reward_std": 0.014490745961666107,
+      "rewards//mean": 0.72625732421875,
+      "rewards//std": 0.04755370691418648,
+      "step": 115
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0232,
+      "grad_norm": 0.5705957412719727,
+      "kl": 0.002923560852650553,
+      "learning_rate": 4.997873014733036e-06,
+      "loss": 0.0001,
+      "num_tokens": 1002832.0,
+      "reward": 0.71453857421875,
+      "reward_std": 0.013323797844350338,
+      "rewards//mean": 0.71453857421875,
+      "rewards//std": 0.05091845244169235,
+      "step": 116
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0234,
+      "grad_norm": 0.45392346382141113,
+      "kl": 0.0019422014956944622,
+      "learning_rate": 4.997807075247147e-06,
+      "loss": 0.0001,
+      "num_tokens": 1011448.0,
+      "reward": 0.70611572265625,
+      "reward_std": 0.015927188098430634,
+      "rewards//mean": 0.70611572265625,
+      "rewards//std": 0.04444960504770279,
+      "step": 117
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0236,
+      "grad_norm": 0.46781232953071594,
+      "kl": 0.0021620240004267544,
+      "learning_rate": 4.9977401296448655e-06,
+      "loss": 0.0001,
+      "num_tokens": 1020104.0,
+      "reward": 0.71502685546875,
+      "reward_std": 0.014722153544425964,
+      "rewards//mean": 0.71502685546875,
+      "rewards//std": 0.04867687448859215,
+      "step": 118
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0238,
+      "grad_norm": 0.46179869771003723,
+      "kl": 0.0022078527545090765,
+      "learning_rate": 4.99767217795316e-06,
+      "loss": 0.0001,
+      "num_tokens": 1028696.0,
+      "reward": 0.72418212890625,
+      "reward_std": 0.011131198145449162,
+      "rewards//mean": 0.72418212890625,
+      "rewards//std": 0.0295663233846426,
+      "step": 119
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.024,
+      "grad_norm": 0.5357843637466431,
+      "kl": 0.002367166889598593,
+      "learning_rate": 4.997603220199399e-06,
+      "loss": 0.0001,
+      "num_tokens": 1037384.0,
+      "reward": 0.7073974609375,
+      "reward_std": 0.013118360191583633,
+      "rewards//mean": 0.7073974609375,
+      "rewards//std": 0.05458429455757141,
+      "step": 120
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0242,
+      "grad_norm": 0.4789181649684906,
+      "kl": 0.0018334937049075961,
+      "learning_rate": 4.99753325641136e-06,
+      "loss": 0.0001,
+      "num_tokens": 1046176.0,
+      "reward": 0.72113037109375,
+      "reward_std": 0.011054251343011856,
+      "rewards//mean": 0.72113037109375,
+      "rewards//std": 0.0613209493458271,
+      "step": 121
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0244,
+      "grad_norm": 0.4643182158470154,
+      "kl": 0.0028950390114914626,
+      "learning_rate": 4.997462286617224e-06,
+      "loss": 0.0001,
+      "num_tokens": 1054816.0,
+      "reward": 0.7144775390625,
+      "reward_std": 0.014479342848062515,
+      "rewards//mean": 0.7144775390625,
+      "rewards//std": 0.06932283937931061,
+      "step": 122
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0246,
+      "grad_norm": 0.45720401406288147,
+      "kl": 0.002508550591301173,
+      "learning_rate": 4.997390310845578e-06,
+      "loss": 0.0001,
+      "num_tokens": 1063496.0,
+      "reward": 0.73211669921875,
+      "reward_std": 0.018719132989645004,
+      "rewards//mean": 0.73211669921875,
+      "rewards//std": 0.043240174651145935,
+      "step": 123
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0248,
+      "grad_norm": 0.4856073260307312,
+      "kl": 0.005021717923227698,
+      "learning_rate": 4.997317329125413e-06,
+      "loss": 0.0002,
+      "num_tokens": 1072104.0,
+      "reward": 0.739501953125,
+      "reward_std": 0.018492162227630615,
+      "rewards//mean": 0.739501953125,
+      "rewards//std": 0.05581790953874588,
+      "step": 124
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.025,
+      "grad_norm": 0.45310959219932556,
+      "kl": 0.0023723691556369886,
+      "learning_rate": 4.997243341486126e-06,
+      "loss": 0.0001,
+      "num_tokens": 1080752.0,
+      "reward": 0.7279052734375,
+      "reward_std": 0.01648058369755745,
+      "rewards//mean": 0.7279052734375,
+      "rewards//std": 0.05334783345460892,
+      "step": 125
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0252,
+      "grad_norm": 0.44085827469825745,
+      "kl": 0.0022444947535404935,
+      "learning_rate": 4.997168347957521e-06,
+      "loss": 0.0001,
+      "num_tokens": 1089368.0,
+      "reward": 0.72052001953125,
+      "reward_std": 0.015059070661664009,
+      "rewards//mean": 0.72052001953125,
+      "rewards//std": 0.0704086497426033,
+      "step": 126
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0254,
+      "grad_norm": 0.47659870982170105,
+      "kl": 0.002819010740495287,
+      "learning_rate": 4.997092348569802e-06,
+      "loss": 0.0001,
+      "num_tokens": 1097992.0,
+      "reward": 0.734130859375,
+      "reward_std": 0.012258566915988922,
+      "rewards//mean": 0.734130859375,
+      "rewards//std": 0.03378882259130478,
+      "step": 127
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0256,
+      "grad_norm": 0.4902152121067047,
+      "kl": 0.0033210097899427637,
+      "learning_rate": 4.9970153433535855e-06,
+      "loss": 0.0001,
+      "num_tokens": 1106576.0,
+      "reward": 0.71600341796875,
+      "reward_std": 0.01741008460521698,
+      "rewards//mean": 0.71600341796875,
+      "rewards//std": 0.04795122891664505,
+      "step": 128
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0258,
+      "grad_norm": 0.467317670583725,
+      "kl": 0.0020708676311187446,
+      "learning_rate": 4.996937332339887e-06,
+      "loss": 0.0001,
+      "num_tokens": 1115160.0,
+      "reward": 0.71319580078125,
+      "reward_std": 0.010551651939749718,
+      "rewards//mean": 0.71319580078125,
+      "rewards//std": 0.04744407534599304,
+      "step": 129
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.026,
+      "grad_norm": 0.5055683851242065,
+      "kl": 0.0025406221684534103,
+      "learning_rate": 4.996858315560129e-06,
+      "loss": 0.0001,
+      "num_tokens": 1123808.0,
+      "reward": 0.72503662109375,
+      "reward_std": 0.015444736927747726,
+      "rewards//mean": 0.72503662109375,
+      "rewards//std": 0.06147873401641846,
+      "step": 130
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0262,
+      "grad_norm": 0.5300055146217346,
+      "kl": 0.004421208694111556,
+      "learning_rate": 4.9967782930461405e-06,
+      "loss": 0.0002,
+      "num_tokens": 1132528.0,
+      "reward": 0.71380615234375,
+      "reward_std": 0.012746734544634819,
+      "rewards//mean": 0.71380615234375,
+      "rewards//std": 0.05404258891940117,
+      "step": 131
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0264,
+      "grad_norm": 0.5882632732391357,
+      "kl": 0.004176836780970916,
+      "learning_rate": 4.9966972648301535e-06,
+      "loss": 0.0002,
+      "num_tokens": 1141160.0,
+      "reward": 0.73260498046875,
+      "reward_std": 0.013695623725652695,
+      "rewards//mean": 0.73260498046875,
+      "rewards//std": 0.0439189113676548,
+      "step": 132
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0266,
+      "grad_norm": 0.5080795288085938,
+      "kl": 0.0037878667353652418,
+      "learning_rate": 4.996615230944808e-06,
+      "loss": 0.0002,
+      "num_tokens": 1149744.0,
+      "reward": 0.7181396484375,
+      "reward_std": 0.021421236917376518,
+      "rewards//mean": 0.7181396484375,
+      "rewards//std": 0.053551748394966125,
+      "step": 133
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0268,
+      "grad_norm": 0.5161112546920776,
+      "kl": 0.002877964361687191,
+      "learning_rate": 4.996532191423145e-06,
+      "loss": 0.0001,
+      "num_tokens": 1158344.0,
+      "reward": 0.71826171875,
+      "reward_std": 0.017237350344657898,
+      "rewards//mean": 0.71826171875,
+      "rewards//std": 0.0640067458152771,
+      "step": 134
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.027,
+      "grad_norm": 0.501823902130127,
+      "kl": 0.003536474294378422,
+      "learning_rate": 4.996448146298615e-06,
+      "loss": 0.0001,
+      "num_tokens": 1166920.0,
+      "reward": 0.68341064453125,
+      "reward_std": 0.01601843349635601,
+      "rewards//mean": 0.68341064453125,
+      "rewards//std": 0.041225045919418335,
+      "step": 135
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0272,
+      "grad_norm": 0.5185138583183289,
+      "kl": 0.004472654749406502,
+      "learning_rate": 4.996363095605069e-06,
+      "loss": 0.0002,
+      "num_tokens": 1175528.0,
+      "reward": 0.7242431640625,
+      "reward_std": 0.018114212900400162,
+      "rewards//mean": 0.7242431640625,
+      "rewards//std": 0.05369402840733528,
+      "step": 136
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0274,
+      "grad_norm": 0.49364981055259705,
+      "kl": 0.003177185819367878,
+      "learning_rate": 4.996277039376767e-06,
+      "loss": 0.0001,
+      "num_tokens": 1184144.0,
+      "reward": 0.72088623046875,
+      "reward_std": 0.01708320900797844,
+      "rewards//mean": 0.72088623046875,
+      "rewards//std": 0.054411906749010086,
+      "step": 137
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0276,
+      "grad_norm": 0.5048626661300659,
+      "kl": 0.003767449234146625,
+      "learning_rate": 4.9961899776483725e-06,
+      "loss": 0.0002,
+      "num_tokens": 1192832.0,
+      "reward": 0.72998046875,
+      "reward_std": 0.013601850718259811,
+      "rewards//mean": 0.72998046875,
+      "rewards//std": 0.047309596091508865,
+      "step": 138
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0278,
+      "grad_norm": 0.5485401153564453,
+      "kl": 0.005610850581433624,
+      "learning_rate": 4.996101910454953e-06,
+      "loss": 0.0002,
+      "num_tokens": 1201472.0,
+      "reward": 0.70831298828125,
+      "reward_std": 0.018774503841996193,
+      "rewards//mean": 0.70831298828125,
+      "rewards//std": 0.07297329604625702,
+      "step": 139
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.028,
+      "grad_norm": 0.5312867164611816,
+      "kl": 0.004881514410953969,
+      "learning_rate": 4.996012837831983e-06,
+      "loss": 0.0002,
+      "num_tokens": 1210176.0,
+      "reward": 0.7449951171875,
+      "reward_std": 0.01512197032570839,
+      "rewards//mean": 0.7449951171875,
+      "rewards//std": 0.04492507874965668,
+      "step": 140
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0282,
+      "grad_norm": 0.7407464981079102,
+      "kl": 0.005604632286122069,
+      "learning_rate": 4.9959227598153395e-06,
+      "loss": 0.0002,
+      "num_tokens": 1218872.0,
+      "reward": 0.7197265625,
+      "reward_std": 0.018171414732933044,
+      "rewards//mean": 0.7197265625,
+      "rewards//std": 0.04184069111943245,
+      "step": 141
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0284,
+      "grad_norm": 0.5207117199897766,
+      "kl": 0.0038259811990428716,
+      "learning_rate": 4.995831676441307e-06,
+      "loss": 0.0002,
+      "num_tokens": 1227624.0,
+      "reward": 0.72515869140625,
+      "reward_std": 0.0129172932356596,
+      "rewards//mean": 0.72515869140625,
+      "rewards//std": 0.047344744205474854,
+      "step": 142
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0286,
+      "grad_norm": 0.4676206111907959,
+      "kl": 0.004619606072083116,
+      "learning_rate": 4.995739587746574e-06,
+      "loss": 0.0002,
+      "num_tokens": 1236264.0,
+      "reward": 0.734375,
+      "reward_std": 0.01602023094892502,
+      "rewards//mean": 0.734375,
+      "rewards//std": 0.04370923712849617,
+      "step": 143
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0288,
+      "grad_norm": 0.4712521731853485,
+      "kl": 0.005641886862576939,
+      "learning_rate": 4.995646493768234e-06,
+      "loss": 0.0002,
+      "num_tokens": 1244984.0,
+      "reward": 0.69970703125,
+      "reward_std": 0.016067516058683395,
+      "rewards//mean": 0.69970703125,
+      "rewards//std": 0.07418610900640488,
+      "step": 144
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.029,
+      "grad_norm": 0.4732939898967743,
+      "kl": 0.003699015636811964,
+      "learning_rate": 4.995552394543784e-06,
+      "loss": 0.0001,
+      "num_tokens": 1253544.0,
+      "reward": 0.68414306640625,
+      "reward_std": 0.017361916601657867,
+      "rewards//mean": 0.68414306640625,
+      "rewards//std": 0.05543471500277519,
+      "step": 145
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0292,
+      "grad_norm": 0.5424770712852478,
+      "kl": 0.0069044766132719815,
+      "learning_rate": 4.995457290111129e-06,
+      "loss": 0.0003,
+      "num_tokens": 1262272.0,
+      "reward": 0.74951171875,
+      "reward_std": 0.014574643224477768,
+      "rewards//mean": 0.74951171875,
+      "rewards//std": 0.048895664513111115,
+      "step": 146
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0294,
+      "grad_norm": 0.5275890231132507,
+      "kl": 0.004333419870818034,
+      "learning_rate": 4.995361180508575e-06,
+      "loss": 0.0002,
+      "num_tokens": 1270984.0,
+      "reward": 0.72943115234375,
+      "reward_std": 0.013785232789814472,
+      "rewards//mean": 0.72943115234375,
+      "rewards//std": 0.055875882506370544,
+      "step": 147
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0296,
+      "grad_norm": 0.5140582919120789,
+      "kl": 0.006125797764980234,
+      "learning_rate": 4.995264065774837e-06,
+      "loss": 0.0002,
+      "num_tokens": 1279664.0,
+      "reward": 0.71893310546875,
+      "reward_std": 0.011785900220274925,
+      "rewards//mean": 0.71893310546875,
+      "rewards//std": 0.05134063959121704,
+      "step": 148
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0298,
+      "grad_norm": 0.5618065595626831,
+      "kl": 0.006264124327572063,
+      "learning_rate": 4.99516594594903e-06,
+      "loss": 0.0003,
+      "num_tokens": 1288360.0,
+      "reward": 0.71807861328125,
+      "reward_std": 0.013035988435149193,
+      "rewards//mean": 0.71807861328125,
+      "rewards//std": 0.053489912301301956,
+      "step": 149
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.03,
+      "grad_norm": 0.502047598361969,
+      "kl": 0.005594079499132931,
+      "learning_rate": 4.9950668210706795e-06,
+      "loss": 0.0002,
+      "num_tokens": 1297032.0,
+      "reward": 0.732177734375,
+      "reward_std": 0.0157606303691864,
+      "rewards//mean": 0.732177734375,
+      "rewards//std": 0.03976839408278465,
+      "step": 150
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0302,
+      "grad_norm": 0.5016326904296875,
+      "kl": 0.00834939838387072,
+      "learning_rate": 4.994966691179712e-06,
+      "loss": 0.0003,
+      "num_tokens": 1305632.0,
+      "reward": 0.703125,
+      "reward_std": 0.014876965433359146,
+      "rewards//mean": 0.703125,
+      "rewards//std": 0.04747569188475609,
+      "step": 151
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0304,
+      "grad_norm": 0.5275666117668152,
+      "kl": 0.006890498974826187,
+      "learning_rate": 4.9948655563164585e-06,
+      "loss": 0.0003,
+      "num_tokens": 1314272.0,
+      "reward": 0.71551513671875,
+      "reward_std": 0.015125017613172531,
+      "rewards//mean": 0.71551513671875,
+      "rewards//std": 0.05302604287862778,
+      "step": 152
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0306,
+      "grad_norm": 0.5499895215034485,
+      "kl": 0.0106024268316105,
+      "learning_rate": 4.994763416521658e-06,
+      "loss": 0.0004,
+      "num_tokens": 1322960.0,
+      "reward": 0.74847412109375,
+      "reward_std": 0.017264600843191147,
+      "rewards//mean": 0.74847412109375,
+      "rewards//std": 0.058874648064374924,
+      "step": 153
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0308,
+      "grad_norm": 0.5384777784347534,
+      "kl": 0.0059372307441663,
+      "learning_rate": 4.994660271836452e-06,
+      "loss": 0.0002,
+      "num_tokens": 1331608.0,
+      "reward": 0.73358154296875,
+      "reward_std": 0.016230447217822075,
+      "rewards//mean": 0.73358154296875,
+      "rewards//std": 0.06771613657474518,
+      "step": 154
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.031,
+      "grad_norm": 0.5435893535614014,
+      "kl": 0.007853654562495649,
+      "learning_rate": 4.994556122302387e-06,
+      "loss": 0.0003,
+      "num_tokens": 1340352.0,
+      "reward": 0.7288818359375,
+      "reward_std": 0.013145558536052704,
+      "rewards//mean": 0.7288818359375,
+      "rewards//std": 0.05553446710109711,
+      "step": 155
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0312,
+      "grad_norm": 0.5240859985351562,
+      "kl": 0.008158890006598085,
+      "learning_rate": 4.994450967961413e-06,
+      "loss": 0.0003,
+      "num_tokens": 1349024.0,
+      "reward": 0.72222900390625,
+      "reward_std": 0.013540119864046574,
+      "rewards//mean": 0.72222900390625,
+      "rewards//std": 0.034443680197000504,
+      "step": 156
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0314,
+      "grad_norm": 0.6642475724220276,
+      "kl": 0.007359111390542239,
+      "learning_rate": 4.994344808855888e-06,
+      "loss": 0.0003,
+      "num_tokens": 1357544.0,
+      "reward": 0.71826171875,
+      "reward_std": 0.018803555518388748,
+      "rewards//mean": 0.71826171875,
+      "rewards//std": 0.05610194429755211,
+      "step": 157
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0316,
+      "grad_norm": 0.52338045835495,
+      "kl": 0.009215345402481034,
+      "learning_rate": 4.994237645028573e-06,
+      "loss": 0.0004,
+      "num_tokens": 1366224.0,
+      "reward": 0.72686767578125,
+      "reward_std": 0.010744665749371052,
+      "rewards//mean": 0.72686767578125,
+      "rewards//std": 0.0492648109793663,
+      "step": 158
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0318,
+      "grad_norm": 0.4993240237236023,
+      "kl": 0.00795084226410836,
+      "learning_rate": 4.994129476522632e-06,
+      "loss": 0.0003,
+      "num_tokens": 1374848.0,
+      "reward": 0.72064208984375,
+      "reward_std": 0.014471987262368202,
+      "rewards//mean": 0.72064208984375,
+      "rewards//std": 0.04946933686733246,
+      "step": 159
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.032,
+      "grad_norm": 0.5304632186889648,
+      "kl": 0.010611457342747599,
+      "learning_rate": 4.994020303381636e-06,
+      "loss": 0.0004,
+      "num_tokens": 1383456.0,
+      "reward": 0.6920166015625,
+      "reward_std": 0.015825100243091583,
+      "rewards//mean": 0.6920166015625,
+      "rewards//std": 0.06738641113042831,
+      "step": 160
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0322,
+      "grad_norm": 0.6055727005004883,
+      "kl": 0.009950865583959967,
+      "learning_rate": 4.993910125649561e-06,
+      "loss": 0.0004,
+      "num_tokens": 1392112.0,
+      "reward": 0.7283935546875,
+      "reward_std": 0.01466367393732071,
+      "rewards//mean": 0.7283935546875,
+      "rewards//std": 0.051102034747600555,
+      "step": 161
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0324,
+      "grad_norm": 0.6653439998626709,
+      "kl": 0.015546724433079362,
+      "learning_rate": 4.993798943370785e-06,
+      "loss": 0.0006,
+      "num_tokens": 1400784.0,
+      "reward": 0.7451171875,
+      "reward_std": 0.014343861490488052,
+      "rewards//mean": 0.7451171875,
+      "rewards//std": 0.0543498657643795,
+      "step": 162
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0326,
+      "grad_norm": 0.5136329531669617,
+      "kl": 0.009563862986396998,
+      "learning_rate": 4.993686756590093e-06,
+      "loss": 0.0004,
+      "num_tokens": 1409344.0,
+      "reward": 0.74176025390625,
+      "reward_std": 0.01264176145195961,
+      "rewards//mean": 0.74176025390625,
+      "rewards//std": 0.031986285001039505,
+      "step": 163
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0328,
+      "grad_norm": 0.6398639678955078,
+      "kl": 0.008364403067389503,
+      "learning_rate": 4.993573565352674e-06,
+      "loss": 0.0003,
+      "num_tokens": 1417920.0,
+      "reward": 0.728271484375,
+      "reward_std": 0.014848753809928894,
+      "rewards//mean": 0.728271484375,
+      "rewards//std": 0.043802645057439804,
+      "step": 164
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.033,
+      "grad_norm": 0.5334981083869934,
+      "kl": 0.015317061392124742,
+      "learning_rate": 4.993459369704121e-06,
+      "loss": 0.0006,
+      "num_tokens": 1426608.0,
+      "reward": 0.71307373046875,
+      "reward_std": 0.014587020501494408,
+      "rewards//mean": 0.71307373046875,
+      "rewards//std": 0.05228722095489502,
+      "step": 165
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0332,
+      "grad_norm": 0.6520506143569946,
+      "kl": 0.014760783000383526,
+      "learning_rate": 4.9933441696904315e-06,
+      "loss": 0.0006,
+      "num_tokens": 1435216.0,
+      "reward": 0.72015380859375,
+      "reward_std": 0.016750846058130264,
+      "rewards//mean": 0.72015380859375,
+      "rewards//std": 0.03944116830825806,
+      "step": 166
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0334,
+      "grad_norm": 0.6047248244285583,
+      "kl": 0.011187054798938334,
+      "learning_rate": 4.993227965358008e-06,
+      "loss": 0.0004,
+      "num_tokens": 1443832.0,
+      "reward": 0.68414306640625,
+      "reward_std": 0.016772452741861343,
+      "rewards//mean": 0.68414306640625,
+      "rewards//std": 0.07010506093502045,
+      "step": 167
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0336,
+      "grad_norm": 0.5380955338478088,
+      "kl": 0.013987196376547217,
+      "learning_rate": 4.99311075675366e-06,
+      "loss": 0.0006,
+      "num_tokens": 1452488.0,
+      "reward": 0.73919677734375,
+      "reward_std": 0.014058486558496952,
+      "rewards//mean": 0.73919677734375,
+      "rewards//std": 0.04323422163724899,
+      "step": 168
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0338,
+      "grad_norm": 0.5614820122718811,
+      "kl": 0.00901414075633511,
+      "learning_rate": 4.992992543924597e-06,
+      "loss": 0.0004,
+      "num_tokens": 1461184.0,
+      "reward": 0.73291015625,
+      "reward_std": 0.015211429446935654,
+      "rewards//mean": 0.73291015625,
+      "rewards//std": 0.052048444747924805,
+      "step": 169
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.034,
+      "grad_norm": 0.5328094363212585,
+      "kl": 0.013381825701799244,
+      "learning_rate": 4.992873326918434e-06,
+      "loss": 0.0005,
+      "num_tokens": 1469920.0,
+      "reward": 0.7449951171875,
+      "reward_std": 0.01563861407339573,
+      "rewards//mean": 0.7449951171875,
+      "rewards//std": 0.05957742780447006,
+      "step": 170
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0342,
+      "grad_norm": 0.568260133266449,
+      "kl": 0.010022971488069743,
+      "learning_rate": 4.992753105783194e-06,
+      "loss": 0.0004,
+      "num_tokens": 1478560.0,
+      "reward": 0.718017578125,
+      "reward_std": 0.01765732280910015,
+      "rewards//mean": 0.718017578125,
+      "rewards//std": 0.058326005935668945,
+      "step": 171
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0344,
+      "grad_norm": 0.6112688183784485,
+      "kl": 0.011806668480858207,
+      "learning_rate": 4.992631880567301e-06,
+      "loss": 0.0005,
+      "num_tokens": 1487232.0,
+      "reward": 0.68988037109375,
+      "reward_std": 0.014870746061205864,
+      "rewards//mean": 0.68988037109375,
+      "rewards//std": 0.060817260295152664,
+      "step": 172
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0346,
+      "grad_norm": 0.5378821492195129,
+      "kl": 0.01460516860242933,
+      "learning_rate": 4.992509651319585e-06,
+      "loss": 0.0006,
+      "num_tokens": 1495848.0,
+      "reward": 0.71533203125,
+      "reward_std": 0.01423371210694313,
+      "rewards//mean": 0.71533203125,
+      "rewards//std": 0.07337196916341782,
+      "step": 173
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0348,
+      "grad_norm": 0.6649944186210632,
+      "kl": 0.014010886079631746,
+      "learning_rate": 4.992386418089279e-06,
+      "loss": 0.0006,
+      "num_tokens": 1504680.0,
+      "reward": 0.7298583984375,
+      "reward_std": 0.012547630816698074,
+      "rewards//mean": 0.7298583984375,
+      "rewards//std": 0.052616409957408905,
+      "step": 174
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.035,
+      "grad_norm": 0.5258428454399109,
+      "kl": 0.014397241990081966,
+      "learning_rate": 4.992262180926022e-06,
+      "loss": 0.0006,
+      "num_tokens": 1513360.0,
+      "reward": 0.706787109375,
+      "reward_std": 0.015542633831501007,
+      "rewards//mean": 0.706787109375,
+      "rewards//std": 0.06885714083909988,
+      "step": 175
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0352,
+      "grad_norm": 0.4891396164894104,
+      "kl": 0.012740209785988554,
+      "learning_rate": 4.992136939879857e-06,
+      "loss": 0.0005,
+      "num_tokens": 1522112.0,
+      "reward": 0.73828125,
+      "reward_std": 0.01267999317497015,
+      "rewards//mean": 0.73828125,
+      "rewards//std": 0.04550643637776375,
+      "step": 176
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0354,
+      "grad_norm": 0.49961039423942566,
+      "kl": 0.01649987616110593,
+      "learning_rate": 4.992010695001229e-06,
+      "loss": 0.0007,
+      "num_tokens": 1530848.0,
+      "reward": 0.7008056640625,
+      "reward_std": 0.012512251734733582,
+      "rewards//mean": 0.7008056640625,
+      "rewards//std": 0.06246790289878845,
+      "step": 177
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0356,
+      "grad_norm": 0.8701191544532776,
+      "kl": 0.019287034403532743,
+      "learning_rate": 4.9918834463409925e-06,
+      "loss": 0.0008,
+      "num_tokens": 1539528.0,
+      "reward": 0.70989990234375,
+      "reward_std": 0.016380542889237404,
+      "rewards//mean": 0.70989990234375,
+      "rewards//std": 0.042210932821035385,
+      "step": 178
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0358,
+      "grad_norm": 0.6032683253288269,
+      "kl": 0.014312484301626682,
+      "learning_rate": 4.991755193950401e-06,
+      "loss": 0.0006,
+      "num_tokens": 1548192.0,
+      "reward": 0.70318603515625,
+      "reward_std": 0.011435788124799728,
+      "rewards//mean": 0.70318603515625,
+      "rewards//std": 0.054831214249134064,
+      "step": 179
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.036,
+      "grad_norm": 0.6240742802619934,
+      "kl": 0.019559299282263964,
+      "learning_rate": 4.991625937881117e-06,
+      "loss": 0.0008,
+      "num_tokens": 1556856.0,
+      "reward": 0.733642578125,
+      "reward_std": 0.015770340338349342,
+      "rewards//mean": 0.733642578125,
+      "rewards//std": 0.05675596743822098,
+      "step": 180
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0362,
+      "grad_norm": 0.6242550611495972,
+      "kl": 0.014271960069891065,
+      "learning_rate": 4.991495678185202e-06,
+      "loss": 0.0006,
+      "num_tokens": 1565488.0,
+      "reward": 0.7296142578125,
+      "reward_std": 0.012346789240837097,
+      "rewards//mean": 0.7296142578125,
+      "rewards//std": 0.07252856343984604,
+      "step": 181
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0364,
+      "grad_norm": 0.6350885629653931,
+      "kl": 0.018139246094506234,
+      "learning_rate": 4.991364414915126e-06,
+      "loss": 0.0007,
+      "num_tokens": 1574184.0,
+      "reward": 0.72760009765625,
+      "reward_std": 0.016285665333271027,
+      "rewards//mean": 0.72760009765625,
+      "rewards//std": 0.06219015270471573,
+      "step": 182
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0366,
+      "grad_norm": 0.640499472618103,
+      "kl": 0.01825080724665895,
+      "learning_rate": 4.9912321481237616e-06,
+      "loss": 0.0007,
+      "num_tokens": 1582888.0,
+      "reward": 0.72503662109375,
+      "reward_std": 0.010994457639753819,
+      "rewards//mean": 0.72503662109375,
+      "rewards//std": 0.03939354792237282,
+      "step": 183
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0368,
+      "grad_norm": 0.5733298659324646,
+      "kl": 0.022698667307849973,
+      "learning_rate": 4.991098877864386e-06,
+      "loss": 0.0009,
+      "num_tokens": 1591408.0,
+      "reward": 0.7388916015625,
+      "reward_std": 0.016481254249811172,
+      "rewards//mean": 0.7388916015625,
+      "rewards//std": 0.043859802186489105,
+      "step": 184
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.037,
+      "grad_norm": 0.54179447889328,
+      "kl": 0.01430770373553969,
+      "learning_rate": 4.99096460419068e-06,
+      "loss": 0.0006,
+      "num_tokens": 1600088.0,
+      "reward": 0.76007080078125,
+      "reward_std": 0.018877511844038963,
+      "rewards//mean": 0.76007080078125,
+      "rewards//std": 0.04165972024202347,
+      "step": 185
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0372,
+      "grad_norm": 0.7062174677848816,
+      "kl": 0.027250012615695596,
+      "learning_rate": 4.990829327156729e-06,
+      "loss": 0.0011,
+      "num_tokens": 1608712.0,
+      "reward": 0.715576171875,
+      "reward_std": 0.012214237824082375,
+      "rewards//mean": 0.715576171875,
+      "rewards//std": 0.052231352776288986,
+      "step": 186
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0374,
+      "grad_norm": 0.6777660846710205,
+      "kl": 0.02611252712085843,
+      "learning_rate": 4.990693046817023e-06,
+      "loss": 0.001,
+      "num_tokens": 1617400.0,
+      "reward": 0.7432861328125,
+      "reward_std": 0.01230183057487011,
+      "rewards//mean": 0.7432861328125,
+      "rewards//std": 0.034481290727853775,
+      "step": 187
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0376,
+      "grad_norm": 0.557637631893158,
+      "kl": 0.018429984629619867,
+      "learning_rate": 4.990555763226456e-06,
+      "loss": 0.0007,
+      "num_tokens": 1625992.0,
+      "reward": 0.74969482421875,
+      "reward_std": 0.01685107871890068,
+      "rewards//mean": 0.74969482421875,
+      "rewards//std": 0.045630306005477905,
+      "step": 188
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0378,
+      "grad_norm": 0.5212416648864746,
+      "kl": 0.017173759290017188,
+      "learning_rate": 4.990417476440326e-06,
+      "loss": 0.0007,
+      "num_tokens": 1634576.0,
+      "reward": 0.70892333984375,
+      "reward_std": 0.014218729920685291,
+      "rewards//mean": 0.70892333984375,
+      "rewards//std": 0.0681646466255188,
+      "step": 189
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.038,
+      "grad_norm": 0.5165665149688721,
+      "kl": 0.015734603686723858,
+      "learning_rate": 4.9902781865143326e-06,
+      "loss": 0.0006,
+      "num_tokens": 1643200.0,
+      "reward": 0.733154296875,
+      "reward_std": 0.014683052897453308,
+      "rewards//mean": 0.733154296875,
+      "rewards//std": 0.047851089388132095,
+      "step": 190
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0382,
+      "grad_norm": 0.683603823184967,
+      "kl": 0.0229703092481941,
+      "learning_rate": 4.990137893504585e-06,
+      "loss": 0.0009,
+      "num_tokens": 1651848.0,
+      "reward": 0.70361328125,
+      "reward_std": 0.014246530830860138,
+      "rewards//mean": 0.70361328125,
+      "rewards//std": 0.047676779329776764,
+      "step": 191
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0384,
+      "grad_norm": 0.5361626148223877,
+      "kl": 0.02232282201293856,
+      "learning_rate": 4.989996597467591e-06,
+      "loss": 0.0009,
+      "num_tokens": 1660472.0,
+      "reward": 0.721923828125,
+      "reward_std": 0.020648304373025894,
+      "rewards//mean": 0.721923828125,
+      "rewards//std": 0.050515249371528625,
+      "step": 192
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0386,
+      "grad_norm": 0.6549078822135925,
+      "kl": 0.028085972415283322,
+      "learning_rate": 4.989854298460265e-06,
+      "loss": 0.0011,
+      "num_tokens": 1669128.0,
+      "reward": 0.73724365234375,
+      "reward_std": 0.015218530781567097,
+      "rewards//mean": 0.73724365234375,
+      "rewards//std": 0.043220214545726776,
+      "step": 193
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0388,
+      "grad_norm": 0.6959535479545593,
+      "kl": 0.03151222656015307,
+      "learning_rate": 4.989710996539926e-06,
+      "loss": 0.0013,
+      "num_tokens": 1677784.0,
+      "reward": 0.748291015625,
+      "reward_std": 0.014268080703914165,
+      "rewards//mean": 0.748291015625,
+      "rewards//std": 0.0479370579123497,
+      "step": 194
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.039,
+      "grad_norm": 0.6917723417282104,
+      "kl": 0.030924884602427483,
+      "learning_rate": 4.989566691764296e-06,
+      "loss": 0.0012,
+      "num_tokens": 1686448.0,
+      "reward": 0.7283935546875,
+      "reward_std": 0.013681046664714813,
+      "rewards//mean": 0.7283935546875,
+      "rewards//std": 0.0343722440302372,
+      "step": 195
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0392,
+      "grad_norm": 0.7647262811660767,
+      "kl": 0.02689244132488966,
+      "learning_rate": 4.9894213841914994e-06,
+      "loss": 0.0011,
+      "num_tokens": 1695072.0,
+      "reward": 0.695068359375,
+      "reward_std": 0.014594485983252525,
+      "rewards//mean": 0.695068359375,
+      "rewards//std": 0.037756409496068954,
+      "step": 196
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0394,
+      "grad_norm": 0.7618040442466736,
+      "kl": 0.03057428600732237,
+      "learning_rate": 4.989275073880067e-06,
+      "loss": 0.0012,
+      "num_tokens": 1703680.0,
+      "reward": 0.7230224609375,
+      "reward_std": 0.018142350018024445,
+      "rewards//mean": 0.7230224609375,
+      "rewards//std": 0.05768650770187378,
+      "step": 197
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0396,
+      "grad_norm": 0.6719552278518677,
+      "kl": 0.040109737077727914,
+      "learning_rate": 4.989127760888932e-06,
+      "loss": 0.0016,
+      "num_tokens": 1712304.0,
+      "reward": 0.721923828125,
+      "reward_std": 0.012543957680463791,
+      "rewards//mean": 0.721923828125,
+      "rewards//std": 0.050654102116823196,
+      "step": 198
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0398,
+      "grad_norm": 0.6652910709381104,
+      "kl": 0.02444028906757012,
+      "learning_rate": 4.988979445277433e-06,
+      "loss": 0.001,
+      "num_tokens": 1720936.0,
+      "reward": 0.70831298828125,
+      "reward_std": 0.01572088897228241,
+      "rewards//mean": 0.70831298828125,
+      "rewards//std": 0.03865145146846771,
+      "step": 199
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.04,
+      "grad_norm": 0.5563658475875854,
+      "kl": 0.021938784746453166,
+      "learning_rate": 4.988830127105312e-06,
+      "loss": 0.0009,
+      "num_tokens": 1729552.0,
+      "reward": 0.71014404296875,
+      "reward_std": 0.01828540489077568,
+      "rewards//mean": 0.71014404296875,
+      "rewards//std": 0.038920748978853226,
+      "step": 200
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0402,
+      "grad_norm": 0.621466338634491,
+      "kl": 0.04022008215542883,
+      "learning_rate": 4.988679806432712e-06,
+      "loss": 0.0016,
+      "num_tokens": 1738184.0,
+      "reward": 0.71197509765625,
+      "reward_std": 0.017939604818820953,
+      "rewards//mean": 0.71197509765625,
+      "rewards//std": 0.047174979001283646,
+      "step": 201
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0404,
+      "grad_norm": 0.7120692729949951,
+      "kl": 0.03540224314201623,
+      "learning_rate": 4.988528483320184e-06,
+      "loss": 0.0014,
+      "num_tokens": 1746792.0,
+      "reward": 0.72320556640625,
+      "reward_std": 0.016589093953371048,
+      "rewards//mean": 0.72320556640625,
+      "rewards//std": 0.04446118324995041,
+      "step": 202
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0406,
+      "grad_norm": 0.6485600471496582,
+      "kl": 0.02893989998847246,
+      "learning_rate": 4.9883761578286805e-06,
+      "loss": 0.0012,
+      "num_tokens": 1755408.0,
+      "reward": 0.73529052734375,
+      "reward_std": 0.015300112776458263,
+      "rewards//mean": 0.73529052734375,
+      "rewards//std": 0.038799602538347244,
+      "step": 203
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0408,
+      "grad_norm": 0.5922731757164001,
+      "kl": 0.034151540603488684,
+      "learning_rate": 4.988222830019559e-06,
+      "loss": 0.0014,
+      "num_tokens": 1764008.0,
+      "reward": 0.73095703125,
+      "reward_std": 0.013814757578074932,
+      "rewards//mean": 0.73095703125,
+      "rewards//std": 0.046106573194265366,
+      "step": 204
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.041,
+      "grad_norm": 0.5785369277000427,
+      "kl": 0.045372437103651464,
+      "learning_rate": 4.988068499954578e-06,
+      "loss": 0.0018,
+      "num_tokens": 1772688.0,
+      "reward": 0.73089599609375,
+      "reward_std": 0.017161503434181213,
+      "rewards//mean": 0.73089599609375,
+      "rewards//std": 0.057122811675071716,
+      "step": 205
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0412,
+      "grad_norm": 0.6962729096412659,
+      "kl": 0.043618743075057864,
+      "learning_rate": 4.987913167695904e-06,
+      "loss": 0.0017,
+      "num_tokens": 1781256.0,
+      "reward": 0.7265625,
+      "reward_std": 0.016600284725427628,
+      "rewards//mean": 0.7265625,
+      "rewards//std": 0.045094750821590424,
+      "step": 206
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0414,
+      "grad_norm": 0.655792772769928,
+      "kl": 0.030919409124180675,
+      "learning_rate": 4.987756833306103e-06,
+      "loss": 0.0012,
+      "num_tokens": 1789976.0,
+      "reward": 0.742431640625,
+      "reward_std": 0.016554612666368484,
+      "rewards//mean": 0.742431640625,
+      "rewards//std": 0.06263887137174606,
+      "step": 207
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0416,
+      "grad_norm": 0.7749574184417725,
+      "kl": 0.0421929273288697,
+      "learning_rate": 4.987599496848147e-06,
+      "loss": 0.0017,
+      "num_tokens": 1798664.0,
+      "reward": 0.72479248046875,
+      "reward_std": 0.014618289656937122,
+      "rewards//mean": 0.72479248046875,
+      "rewards//std": 0.05295005068182945,
+      "step": 208
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0418,
+      "grad_norm": 0.759087324142456,
+      "kl": 0.04364440473727882,
+      "learning_rate": 4.987441158385411e-06,
+      "loss": 0.0017,
+      "num_tokens": 1807424.0,
+      "reward": 0.68939208984375,
+      "reward_std": 0.012192411348223686,
+      "rewards//mean": 0.68939208984375,
+      "rewards//std": 0.06599829345941544,
+      "step": 209
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.042,
+      "grad_norm": 0.6689143776893616,
+      "kl": 0.05778496875427663,
+      "learning_rate": 4.987281817981674e-06,
+      "loss": 0.0023,
+      "num_tokens": 1816088.0,
+      "reward": 0.72271728515625,
+      "reward_std": 0.014015286229550838,
+      "rewards//mean": 0.72271728515625,
+      "rewards//std": 0.049290310591459274,
+      "step": 210
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0422,
+      "grad_norm": 0.666499674320221,
+      "kl": 0.043112656101584435,
+      "learning_rate": 4.987121475701118e-06,
+      "loss": 0.0017,
+      "num_tokens": 1824672.0,
+      "reward": 0.69085693359375,
+      "reward_std": 0.013759355992078781,
+      "rewards//mean": 0.69085693359375,
+      "rewards//std": 0.06431391090154648,
+      "step": 211
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0424,
+      "grad_norm": 0.6885657906532288,
+      "kl": 0.039523816551081836,
+      "learning_rate": 4.986960131608329e-06,
+      "loss": 0.0016,
+      "num_tokens": 1833248.0,
+      "reward": 0.76361083984375,
+      "reward_std": 0.021125439554452896,
+      "rewards//mean": 0.76361083984375,
+      "rewards//std": 0.03490383177995682,
+      "step": 212
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0426,
+      "grad_norm": 0.7056341171264648,
+      "kl": 0.03723469818942249,
+      "learning_rate": 4.986797785768296e-06,
+      "loss": 0.0015,
+      "num_tokens": 1841808.0,
+      "reward": 0.73406982421875,
+      "reward_std": 0.013552498072385788,
+      "rewards//mean": 0.73406982421875,
+      "rewards//std": 0.058044735342264175,
+      "step": 213
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0428,
+      "grad_norm": 0.5871314406394958,
+      "kl": 0.03060774417826906,
+      "learning_rate": 4.986634438246413e-06,
+      "loss": 0.0012,
+      "num_tokens": 1850392.0,
+      "reward": 0.7166748046875,
+      "reward_std": 0.018129628151655197,
+      "rewards//mean": 0.7166748046875,
+      "rewards//std": 0.05835960432887077,
+      "step": 214
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.043,
+      "grad_norm": 0.556708812713623,
+      "kl": 0.02814829646376893,
+      "learning_rate": 4.986470089108476e-06,
+      "loss": 0.0011,
+      "num_tokens": 1859016.0,
+      "reward": 0.7529296875,
+      "reward_std": 0.017662834376096725,
+      "rewards//mean": 0.7529296875,
+      "rewards//std": 0.045255593955516815,
+      "step": 215
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0432,
+      "grad_norm": 0.771795928478241,
+      "kl": 0.062261566519737244,
+      "learning_rate": 4.986304738420684e-06,
+      "loss": 0.0025,
+      "num_tokens": 1867616.0,
+      "reward": 0.7291259765625,
+      "reward_std": 0.009812846779823303,
+      "rewards//mean": 0.7291259765625,
+      "rewards//std": 0.02753545716404915,
+      "step": 216
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0434,
+      "grad_norm": 0.6160389184951782,
+      "kl": 0.04629370174370706,
+      "learning_rate": 4.986138386249641e-06,
+      "loss": 0.0019,
+      "num_tokens": 1876296.0,
+      "reward": 0.7247314453125,
+      "reward_std": 0.018973447382450104,
+      "rewards//mean": 0.7247314453125,
+      "rewards//std": 0.04735804721713066,
+      "step": 217
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0436,
+      "grad_norm": 0.6583981513977051,
+      "kl": 0.04990657512098551,
+      "learning_rate": 4.985971032662352e-06,
+      "loss": 0.002,
+      "num_tokens": 1884904.0,
+      "reward": 0.7359619140625,
+      "reward_std": 0.017147067934274673,
+      "rewards//mean": 0.7359619140625,
+      "rewards//std": 0.04669753462076187,
+      "step": 218
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0438,
+      "grad_norm": 0.8134801983833313,
+      "kl": 0.0423719760729,
+      "learning_rate": 4.98580267772623e-06,
+      "loss": 0.0017,
+      "num_tokens": 1893616.0,
+      "reward": 0.72003173828125,
+      "reward_std": 0.016933666542172432,
+      "rewards//mean": 0.72003173828125,
+      "rewards//std": 0.06031915172934532,
+      "step": 219
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.044,
+      "grad_norm": 0.6741657853126526,
+      "kl": 0.03965398599393666,
+      "learning_rate": 4.985633321509086e-06,
+      "loss": 0.0016,
+      "num_tokens": 1902160.0,
+      "reward": 0.68719482421875,
+      "reward_std": 0.015195935033261776,
+      "rewards//mean": 0.68719482421875,
+      "rewards//std": 0.05944652855396271,
+      "step": 220
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0442,
+      "grad_norm": 0.9493129849433899,
+      "kl": 0.046657787868753076,
+      "learning_rate": 4.985462964079137e-06,
+      "loss": 0.0019,
+      "num_tokens": 1910880.0,
+      "reward": 0.71942138671875,
+      "reward_std": 0.016195297241210938,
+      "rewards//mean": 0.71942138671875,
+      "rewards//std": 0.05246121808886528,
+      "step": 221
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0444,
+      "grad_norm": 0.7087270021438599,
+      "kl": 0.04575832257978618,
+      "learning_rate": 4.985291605505004e-06,
+      "loss": 0.0018,
+      "num_tokens": 1919640.0,
+      "reward": 0.702392578125,
+      "reward_std": 0.0167884211987257,
+      "rewards//mean": 0.702392578125,
+      "rewards//std": 0.05223599076271057,
+      "step": 222
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0446,
+      "grad_norm": 0.6752747297286987,
+      "kl": 0.04726569773629308,
+      "learning_rate": 4.9851192458557084e-06,
+      "loss": 0.0019,
+      "num_tokens": 1928296.0,
+      "reward": 0.71807861328125,
+      "reward_std": 0.01220088079571724,
+      "rewards//mean": 0.71807861328125,
+      "rewards//std": 0.037438686937093735,
+      "step": 223
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0448,
+      "grad_norm": 0.7452352046966553,
+      "kl": 0.05152314284350723,
+      "learning_rate": 4.984945885200679e-06,
+      "loss": 0.0021,
+      "num_tokens": 1937088.0,
+      "reward": 0.74041748046875,
+      "reward_std": 0.013437759131193161,
+      "rewards//mean": 0.74041748046875,
+      "rewards//std": 0.040072083473205566,
+      "step": 224
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.045,
+      "grad_norm": 0.6877680420875549,
+      "kl": 0.051004784647375345,
+      "learning_rate": 4.984771523609744e-06,
+      "loss": 0.002,
+      "num_tokens": 1945688.0,
+      "reward": 0.73846435546875,
+      "reward_std": 0.016483765095472336,
+      "rewards//mean": 0.73846435546875,
+      "rewards//std": 0.0488903634250164,
+      "step": 225
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0452,
+      "grad_norm": 0.7118944525718689,
+      "kl": 0.05141365760937333,
+      "learning_rate": 4.9845961611531356e-06,
+      "loss": 0.0021,
+      "num_tokens": 1954320.0,
+      "reward": 0.74041748046875,
+      "reward_std": 0.015977714210748672,
+      "rewards//mean": 0.74041748046875,
+      "rewards//std": 0.05143725872039795,
+      "step": 226
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0454,
+      "grad_norm": 0.9068376421928406,
+      "kl": 0.05422702501527965,
+      "learning_rate": 4.984419797901491e-06,
+      "loss": 0.0022,
+      "num_tokens": 1962944.0,
+      "reward": 0.71044921875,
+      "reward_std": 0.01507932785898447,
+      "rewards//mean": 0.71044921875,
+      "rewards//std": 0.036301881074905396,
+      "step": 227
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0456,
+      "grad_norm": 0.7399264574050903,
+      "kl": 0.05034496798180044,
+      "learning_rate": 4.984242433925849e-06,
+      "loss": 0.002,
+      "num_tokens": 1971624.0,
+      "reward": 0.75054931640625,
+      "reward_std": 0.015884902328252792,
+      "rewards//mean": 0.75054931640625,
+      "rewards//std": 0.06389451771974564,
+      "step": 228
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0458,
+      "grad_norm": 0.7348447442054749,
+      "kl": 0.049029057379812,
+      "learning_rate": 4.984064069297652e-06,
+      "loss": 0.002,
+      "num_tokens": 1980280.0,
+      "reward": 0.697021484375,
+      "reward_std": 0.015012217685580254,
+      "rewards//mean": 0.697021484375,
+      "rewards//std": 0.056743159890174866,
+      "step": 229
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.046,
+      "grad_norm": 0.7991431355476379,
+      "kl": 0.07105646608397365,
+      "learning_rate": 4.983884704088745e-06,
+      "loss": 0.0028,
+      "num_tokens": 1988880.0,
+      "reward": 0.695068359375,
+      "reward_std": 0.01788986101746559,
+      "rewards//mean": 0.695068359375,
+      "rewards//std": 0.06029834970831871,
+      "step": 230
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0462,
+      "grad_norm": 0.6498074531555176,
+      "kl": 0.05387417250312865,
+      "learning_rate": 4.983704338371375e-06,
+      "loss": 0.0022,
+      "num_tokens": 1997536.0,
+      "reward": 0.738037109375,
+      "reward_std": 0.012314668856561184,
+      "rewards//mean": 0.738037109375,
+      "rewards//std": 0.05002864450216293,
+      "step": 231
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0464,
+      "grad_norm": 0.6092854142189026,
+      "kl": 0.04524057498201728,
+      "learning_rate": 4.983522972218196e-06,
+      "loss": 0.0018,
+      "num_tokens": 2006280.0,
+      "reward": 0.7039794921875,
+      "reward_std": 0.013910917565226555,
+      "rewards//mean": 0.7039794921875,
+      "rewards//std": 0.04475763440132141,
+      "step": 232
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0466,
+      "grad_norm": 0.5306748747825623,
+      "kl": 0.047080037416890264,
+      "learning_rate": 4.983340605702261e-06,
+      "loss": 0.0019,
+      "num_tokens": 2015040.0,
+      "reward": 0.7469482421875,
+      "reward_std": 0.011357417330145836,
+      "rewards//mean": 0.7469482421875,
+      "rewards//std": 0.03777945414185524,
+      "step": 233
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0468,
+      "grad_norm": 0.768358588218689,
+      "kl": 0.03969805908855051,
+      "learning_rate": 4.983157238897026e-06,
+      "loss": 0.0016,
+      "num_tokens": 2023640.0,
+      "reward": 0.7327880859375,
+      "reward_std": 0.015326134860515594,
+      "rewards//mean": 0.7327880859375,
+      "rewards//std": 0.04921766370534897,
+      "step": 234
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.047,
+      "grad_norm": 0.6886104941368103,
+      "kl": 0.05683159315958619,
+      "learning_rate": 4.982972871876353e-06,
+      "loss": 0.0023,
+      "num_tokens": 2032216.0,
+      "reward": 0.7303466796875,
+      "reward_std": 0.010093173943459988,
+      "rewards//mean": 0.7303466796875,
+      "rewards//std": 0.046449214220047,
+      "step": 235
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0472,
+      "grad_norm": 0.5981897711753845,
+      "kl": 0.04994144011288881,
+      "learning_rate": 4.982787504714503e-06,
+      "loss": 0.002,
+      "num_tokens": 2040856.0,
+      "reward": 0.739501953125,
+      "reward_std": 0.018706027418375015,
+      "rewards//mean": 0.739501953125,
+      "rewards//std": 0.05437158793210983,
+      "step": 236
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0474,
+      "grad_norm": 0.7223436832427979,
+      "kl": 0.06479091383516788,
+      "learning_rate": 4.982601137486144e-06,
+      "loss": 0.0026,
+      "num_tokens": 2049408.0,
+      "reward": 0.71697998046875,
+      "reward_std": 0.014704955741763115,
+      "rewards//mean": 0.71697998046875,
+      "rewards//std": 0.0494886115193367,
+      "step": 237
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0476,
+      "grad_norm": 0.6980536580085754,
+      "kl": 0.04527810198487714,
+      "learning_rate": 4.9824137702663424e-06,
+      "loss": 0.0018,
+      "num_tokens": 2058016.0,
+      "reward": 0.71221923828125,
+      "reward_std": 0.01778585836291313,
+      "rewards//mean": 0.71221923828125,
+      "rewards//std": 0.05478951334953308,
+      "step": 238
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0478,
+      "grad_norm": 1.1460226774215698,
+      "kl": 0.06317855324596167,
+      "learning_rate": 4.982225403130572e-06,
+      "loss": 0.0025,
+      "num_tokens": 2066648.0,
+      "reward": 0.73974609375,
+      "reward_std": 0.01631517894566059,
+      "rewards//mean": 0.73974609375,
+      "rewards//std": 0.03925115987658501,
+      "step": 239
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.048,
+      "grad_norm": 0.9314346313476562,
+      "kl": 0.0637447414919734,
+      "learning_rate": 4.982036036154706e-06,
+      "loss": 0.0025,
+      "num_tokens": 2075336.0,
+      "reward": 0.71893310546875,
+      "reward_std": 0.013403713703155518,
+      "rewards//mean": 0.71893310546875,
+      "rewards//std": 0.05558551847934723,
+      "step": 240
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0482,
+      "grad_norm": 0.6399586200714111,
+      "kl": 0.04521946748718619,
+      "learning_rate": 4.981845669415022e-06,
+      "loss": 0.0018,
+      "num_tokens": 2083952.0,
+      "reward": 0.710693359375,
+      "reward_std": 0.017721328884363174,
+      "rewards//mean": 0.710693359375,
+      "rewards//std": 0.032037414610385895,
+      "step": 241
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0484,
+      "grad_norm": 0.7257511019706726,
+      "kl": 0.04954640497453511,
+      "learning_rate": 4.981654302988198e-06,
+      "loss": 0.002,
+      "num_tokens": 2092624.0,
+      "reward": 0.756591796875,
+      "reward_std": 0.014372417703270912,
+      "rewards//mean": 0.756591796875,
+      "rewards//std": 0.04212266206741333,
+      "step": 242
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0486,
+      "grad_norm": 0.7490444183349609,
+      "kl": 0.06151878600940108,
+      "learning_rate": 4.9814619369513184e-06,
+      "loss": 0.0025,
+      "num_tokens": 2101160.0,
+      "reward": 0.7039794921875,
+      "reward_std": 0.01579831726849079,
+      "rewards//mean": 0.7039794921875,
+      "rewards//std": 0.06837564706802368,
+      "step": 243
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0488,
+      "grad_norm": 0.7180415391921997,
+      "kl": 0.041042948490940034,
+      "learning_rate": 4.981268571381867e-06,
+      "loss": 0.0016,
+      "num_tokens": 2109792.0,
+      "reward": 0.732421875,
+      "reward_std": 0.015042596496641636,
+      "rewards//mean": 0.732421875,
+      "rewards//std": 0.04142765328288078,
+      "step": 244
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.049,
+      "grad_norm": 0.7131726145744324,
+      "kl": 0.04773534648120403,
+      "learning_rate": 4.981074206357732e-06,
+      "loss": 0.0019,
+      "num_tokens": 2118552.0,
+      "reward": 0.7369384765625,
+      "reward_std": 0.011528298258781433,
+      "rewards//mean": 0.7369384765625,
+      "rewards//std": 0.04968539997935295,
+      "step": 245
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0492,
+      "grad_norm": 0.775935709476471,
+      "kl": 0.0707710012793541,
+      "learning_rate": 4.980878841957203e-06,
+      "loss": 0.0028,
+      "num_tokens": 2127248.0,
+      "reward": 0.69036865234375,
+      "reward_std": 0.012703044340014458,
+      "rewards//mean": 0.69036865234375,
+      "rewards//std": 0.05450669303536415,
+      "step": 246
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0494,
+      "grad_norm": 0.7658054232597351,
+      "kl": 0.0640805927105248,
+      "learning_rate": 4.980682478258973e-06,
+      "loss": 0.0026,
+      "num_tokens": 2135896.0,
+      "reward": 0.72894287109375,
+      "reward_std": 0.01790812611579895,
+      "rewards//mean": 0.72894287109375,
+      "rewards//std": 0.05467694625258446,
+      "step": 247
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0496,
+      "grad_norm": 0.5831694006919861,
+      "kl": 0.04178021801635623,
+      "learning_rate": 4.980485115342138e-06,
+      "loss": 0.0017,
+      "num_tokens": 2144536.0,
+      "reward": 0.70562744140625,
+      "reward_std": 0.019685041159391403,
+      "rewards//mean": 0.70562744140625,
+      "rewards//std": 0.06453383713960648,
+      "step": 248
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0498,
+      "grad_norm": 0.686213493347168,
+      "kl": 0.05178135330788791,
+      "learning_rate": 4.980286753286196e-06,
+      "loss": 0.0021,
+      "num_tokens": 2153104.0,
+      "reward": 0.73675537109375,
+      "reward_std": 0.015449654310941696,
+      "rewards//mean": 0.73675537109375,
+      "rewards//std": 0.022234952077269554,
+      "step": 249
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.05,
+      "grad_norm": 0.5769968032836914,
+      "kl": 0.04894346767105162,
+      "learning_rate": 4.980087392171045e-06,
+      "loss": 0.002,
+      "num_tokens": 2161736.0,
+      "reward": 0.7308349609375,
+      "reward_std": 0.009863736107945442,
+      "rewards//mean": 0.7308349609375,
+      "rewards//std": 0.04522731900215149,
+      "step": 250
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0502,
+      "grad_norm": 0.6053898930549622,
+      "kl": 0.04904163209721446,
+      "learning_rate": 4.9798870320769884e-06,
+      "loss": 0.002,
+      "num_tokens": 2170344.0,
+      "reward": 0.71343994140625,
+      "reward_std": 0.011166264303028584,
+      "rewards//mean": 0.71343994140625,
+      "rewards//std": 0.05631681904196739,
+      "step": 251
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0504,
+      "grad_norm": 0.7947268486022949,
+      "kl": 0.0519161622505635,
+      "learning_rate": 4.979685673084733e-06,
+      "loss": 0.0021,
+      "num_tokens": 2179056.0,
+      "reward": 0.77587890625,
+      "reward_std": 0.014476396143436432,
+      "rewards//mean": 0.77587890625,
+      "rewards//std": 0.032057251781225204,
+      "step": 252
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0506,
+      "grad_norm": 0.693539559841156,
+      "kl": 0.05135406623594463,
+      "learning_rate": 4.979483315275385e-06,
+      "loss": 0.0021,
+      "num_tokens": 2187640.0,
+      "reward": 0.70477294921875,
+      "reward_std": 0.012753374874591827,
+      "rewards//mean": 0.70477294921875,
+      "rewards//std": 0.05521116405725479,
+      "step": 253
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0508,
+      "grad_norm": 0.6642072796821594,
+      "kl": 0.07001969963312149,
+      "learning_rate": 4.979279958730454e-06,
+      "loss": 0.0028,
+      "num_tokens": 2196336.0,
+      "reward": 0.74652099609375,
+      "reward_std": 0.012642599642276764,
+      "rewards//mean": 0.74652099609375,
+      "rewards//std": 0.036172330379486084,
+      "step": 254
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.051,
+      "grad_norm": 0.7698396444320679,
+      "kl": 0.05689065787009895,
+      "learning_rate": 4.979075603531852e-06,
+      "loss": 0.0023,
+      "num_tokens": 2205024.0,
+      "reward": 0.7344970703125,
+      "reward_std": 0.014306074939668179,
+      "rewards//mean": 0.7344970703125,
+      "rewards//std": 0.04307273030281067,
+      "step": 255
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0512,
+      "grad_norm": 0.6658589839935303,
+      "kl": 0.04703840473666787,
+      "learning_rate": 4.978870249761893e-06,
+      "loss": 0.0019,
+      "num_tokens": 2213608.0,
+      "reward": 0.74749755859375,
+      "reward_std": 0.01808979921042919,
+      "rewards//mean": 0.74749755859375,
+      "rewards//std": 0.06493417918682098,
+      "step": 256
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0514,
+      "grad_norm": 0.5617851614952087,
+      "kl": 0.04335748380981386,
+      "learning_rate": 4.978663897503294e-06,
+      "loss": 0.0017,
+      "num_tokens": 2222264.0,
+      "reward": 0.76544189453125,
+      "reward_std": 0.01435675285756588,
+      "rewards//mean": 0.76544189453125,
+      "rewards//std": 0.05317598953843117,
+      "step": 257
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0516,
+      "grad_norm": 0.7687885761260986,
+      "kl": 0.06642695516347885,
+      "learning_rate": 4.978456546839175e-06,
+      "loss": 0.0027,
+      "num_tokens": 2230880.0,
+      "reward": 0.71099853515625,
+      "reward_std": 0.01172902062535286,
+      "rewards//mean": 0.71099853515625,
+      "rewards//std": 0.045694950968027115,
+      "step": 258
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0518,
+      "grad_norm": 0.747726559638977,
+      "kl": 0.05708181764930487,
+      "learning_rate": 4.978248197853053e-06,
+      "loss": 0.0023,
+      "num_tokens": 2239608.0,
+      "reward": 0.76300048828125,
+      "reward_std": 0.013816908933222294,
+      "rewards//mean": 0.76300048828125,
+      "rewards//std": 0.04265507683157921,
+      "step": 259
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.052,
+      "grad_norm": 0.7513110041618347,
+      "kl": 0.06573500973172486,
+      "learning_rate": 4.978038850628855e-06,
+      "loss": 0.0026,
+      "num_tokens": 2248296.0,
+      "reward": 0.73321533203125,
+      "reward_std": 0.012971704825758934,
+      "rewards//mean": 0.73321533203125,
+      "rewards//std": 0.03936932981014252,
+      "step": 260
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0522,
+      "grad_norm": 0.8820503354072571,
+      "kl": 0.08888540370389819,
+      "learning_rate": 4.977828505250903e-06,
+      "loss": 0.0036,
+      "num_tokens": 2256976.0,
+      "reward": 0.71124267578125,
+      "reward_std": 0.015261702239513397,
+      "rewards//mean": 0.71124267578125,
+      "rewards//std": 0.048025041818618774,
+      "step": 261
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0524,
+      "grad_norm": 0.5689162611961365,
+      "kl": 0.058710358338430524,
+      "learning_rate": 4.977617161803927e-06,
+      "loss": 0.0023,
+      "num_tokens": 2265672.0,
+      "reward": 0.766357421875,
+      "reward_std": 0.013638041913509369,
+      "rewards//mean": 0.766357421875,
+      "rewards//std": 0.047856152057647705,
+      "step": 262
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0526,
+      "grad_norm": 0.7432250380516052,
+      "kl": 0.05571276368573308,
+      "learning_rate": 4.977404820373053e-06,
+      "loss": 0.0022,
+      "num_tokens": 2274336.0,
+      "reward": 0.75738525390625,
+      "reward_std": 0.012658031657338142,
+      "rewards//mean": 0.75738525390625,
+      "rewards//std": 0.04126431792974472,
+      "step": 263
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0528,
+      "grad_norm": 0.5365133881568909,
+      "kl": 0.06589075829833746,
+      "learning_rate": 4.977191481043814e-06,
+      "loss": 0.0026,
+      "num_tokens": 2282984.0,
+      "reward": 0.7647705078125,
+      "reward_std": 0.012238910421729088,
+      "rewards//mean": 0.7647705078125,
+      "rewards//std": 0.036798667162656784,
+      "step": 264
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.053,
+      "grad_norm": 0.7748925089836121,
+      "kl": 0.06631747842766345,
+      "learning_rate": 4.976977143902143e-06,
+      "loss": 0.0027,
+      "num_tokens": 2291520.0,
+      "reward": 0.71173095703125,
+      "reward_std": 0.01299564354121685,
+      "rewards//mean": 0.71173095703125,
+      "rewards//std": 0.05950303003191948,
+      "step": 265
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0532,
+      "grad_norm": 0.6161185503005981,
+      "kl": 0.07089789980091155,
+      "learning_rate": 4.976761809034375e-06,
+      "loss": 0.0028,
+      "num_tokens": 2300080.0,
+      "reward": 0.71710205078125,
+      "reward_std": 0.015387684106826782,
+      "rewards//mean": 0.71710205078125,
+      "rewards//std": 0.03893085941672325,
+      "step": 266
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0534,
+      "grad_norm": 0.91294926404953,
+      "kl": 0.10131165734492242,
+      "learning_rate": 4.976545476527246e-06,
+      "loss": 0.0041,
+      "num_tokens": 2308656.0,
+      "reward": 0.72802734375,
+      "reward_std": 0.01398141123354435,
+      "rewards//mean": 0.72802734375,
+      "rewards//std": 0.042606472969055176,
+      "step": 267
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0536,
+      "grad_norm": 0.7760770320892334,
+      "kl": 0.07038976019248366,
+      "learning_rate": 4.976328146467895e-06,
+      "loss": 0.0028,
+      "num_tokens": 2317368.0,
+      "reward": 0.75408935546875,
+      "reward_std": 0.017939291894435883,
+      "rewards//mean": 0.75408935546875,
+      "rewards//std": 0.04169821739196777,
+      "step": 268
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0538,
+      "grad_norm": 0.7210671305656433,
+      "kl": 0.07773464918136597,
+      "learning_rate": 4.976109818943863e-06,
+      "loss": 0.0031,
+      "num_tokens": 2325992.0,
+      "reward": 0.7349853515625,
+      "reward_std": 0.013170282356441021,
+      "rewards//mean": 0.7349853515625,
+      "rewards//std": 0.03823667764663696,
+      "step": 269
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.054,
+      "grad_norm": 0.7028035521507263,
+      "kl": 0.0827886825427413,
+      "learning_rate": 4.975890494043092e-06,
+      "loss": 0.0033,
+      "num_tokens": 2334616.0,
+      "reward": 0.75750732421875,
+      "reward_std": 0.013806039467453957,
+      "rewards//mean": 0.75750732421875,
+      "rewards//std": 0.03447311371564865,
+      "step": 270
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0542,
+      "grad_norm": 0.6302931904792786,
+      "kl": 0.07922178274020553,
+      "learning_rate": 4.975670171853926e-06,
+      "loss": 0.0032,
+      "num_tokens": 2343264.0,
+      "reward": 0.7408447265625,
+      "reward_std": 0.01199992373585701,
+      "rewards//mean": 0.7408447265625,
+      "rewards//std": 0.03118186816573143,
+      "step": 271
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0544,
+      "grad_norm": 0.5366842746734619,
+      "kl": 0.053182843839749694,
+      "learning_rate": 4.975448852465111e-06,
+      "loss": 0.0021,
+      "num_tokens": 2351936.0,
+      "reward": 0.73406982421875,
+      "reward_std": 0.012117596343159676,
+      "rewards//mean": 0.73406982421875,
+      "rewards//std": 0.05836515128612518,
+      "step": 272
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0546,
+      "grad_norm": 0.7956821918487549,
+      "kl": 0.10613921284675598,
+      "learning_rate": 4.975226535965795e-06,
+      "loss": 0.0042,
+      "num_tokens": 2360560.0,
+      "reward": 0.73834228515625,
+      "reward_std": 0.013946297578513622,
+      "rewards//mean": 0.73834228515625,
+      "rewards//std": 0.045477114617824554,
+      "step": 273
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0548,
+      "grad_norm": 0.825093686580658,
+      "kl": 0.06953645718749613,
+      "learning_rate": 4.975003222445525e-06,
+      "loss": 0.0028,
+      "num_tokens": 2369160.0,
+      "reward": 0.69384765625,
+      "reward_std": 0.0194076094776392,
+      "rewards//mean": 0.69384765625,
+      "rewards//std": 0.048462796956300735,
+      "step": 274
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.055,
+      "grad_norm": 0.7195761799812317,
+      "kl": 0.06491784797981381,
+      "learning_rate": 4.974778911994254e-06,
+      "loss": 0.0026,
+      "num_tokens": 2377840.0,
+      "reward": 0.7286376953125,
+      "reward_std": 0.015999507158994675,
+      "rewards//mean": 0.7286376953125,
+      "rewards//std": 0.04492238163948059,
+      "step": 275
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0552,
+      "grad_norm": 0.6829413175582886,
+      "kl": 0.06265947036445141,
+      "learning_rate": 4.974553604702332e-06,
+      "loss": 0.0025,
+      "num_tokens": 2386504.0,
+      "reward": 0.74267578125,
+      "reward_std": 0.016472984105348587,
+      "rewards//mean": 0.74267578125,
+      "rewards//std": 0.03249996155500412,
+      "step": 276
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0554,
+      "grad_norm": 0.8279444575309753,
+      "kl": 0.109421216417104,
+      "learning_rate": 4.974327300660515e-06,
+      "loss": 0.0044,
+      "num_tokens": 2395128.0,
+      "reward": 0.74176025390625,
+      "reward_std": 0.013287878595292568,
+      "rewards//mean": 0.74176025390625,
+      "rewards//std": 0.03989148512482643,
+      "step": 277
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0556,
+      "grad_norm": 0.8692975640296936,
+      "kl": 0.13027487508952618,
+      "learning_rate": 4.974099999959957e-06,
+      "loss": 0.0052,
+      "num_tokens": 2403824.0,
+      "reward": 0.71258544921875,
+      "reward_std": 0.01673486828804016,
+      "rewards//mean": 0.71258544921875,
+      "rewards//std": 0.045876458287239075,
+      "step": 278
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0558,
+      "grad_norm": 0.6421030759811401,
+      "kl": 0.075614667031914,
+      "learning_rate": 4.973871702692215e-06,
+      "loss": 0.003,
+      "num_tokens": 2412440.0,
+      "reward": 0.72467041015625,
+      "reward_std": 0.015497658401727676,
+      "rewards//mean": 0.72467041015625,
+      "rewards//std": 0.04439099133014679,
+      "step": 279
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.056,
+      "grad_norm": 0.7968896627426147,
+      "kl": 0.08366691786795855,
+      "learning_rate": 4.973642408949247e-06,
+      "loss": 0.0033,
+      "num_tokens": 2421072.0,
+      "reward": 0.693115234375,
+      "reward_std": 0.013288882561028004,
+      "rewards//mean": 0.693115234375,
+      "rewards//std": 0.04501882195472717,
+      "step": 280
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0562,
+      "grad_norm": 0.8614963889122009,
+      "kl": 0.10154542187228799,
+      "learning_rate": 4.9734121188234115e-06,
+      "loss": 0.0041,
+      "num_tokens": 2429736.0,
+      "reward": 0.72857666015625,
+      "reward_std": 0.013583678752183914,
+      "rewards//mean": 0.72857666015625,
+      "rewards//std": 0.04098678007721901,
+      "step": 281
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0564,
+      "grad_norm": 1.0249260663986206,
+      "kl": 0.10393307078629732,
+      "learning_rate": 4.973180832407471e-06,
+      "loss": 0.0042,
+      "num_tokens": 2438336.0,
+      "reward": 0.72576904296875,
+      "reward_std": 0.01790856570005417,
+      "rewards//mean": 0.72576904296875,
+      "rewards//std": 0.03053005412220955,
+      "step": 282
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0566,
+      "grad_norm": 0.8311775922775269,
+      "kl": 0.11601218301802874,
+      "learning_rate": 4.972948549794587e-06,
+      "loss": 0.0046,
+      "num_tokens": 2446944.0,
+      "reward": 0.7076416015625,
+      "reward_std": 0.017834939062595367,
+      "rewards//mean": 0.7076416015625,
+      "rewards//std": 0.036632098257541656,
+      "step": 283
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0568,
+      "grad_norm": 0.8473641872406006,
+      "kl": 0.12403094908222556,
+      "learning_rate": 4.972715271078323e-06,
+      "loss": 0.005,
+      "num_tokens": 2455528.0,
+      "reward": 0.748779296875,
+      "reward_std": 0.014438275247812271,
+      "rewards//mean": 0.748779296875,
+      "rewards//std": 0.036623213440179825,
+      "step": 284
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.057,
+      "grad_norm": 1.0325453281402588,
+      "kl": 0.15802243910729885,
+      "learning_rate": 4.972480996352644e-06,
+      "loss": 0.0063,
+      "num_tokens": 2464104.0,
+      "reward": 0.72064208984375,
+      "reward_std": 0.011821886524558067,
+      "rewards//mean": 0.72064208984375,
+      "rewards//std": 0.04506784677505493,
+      "step": 285
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0572,
+      "grad_norm": 0.8279632329940796,
+      "kl": 0.14361037500202656,
+      "learning_rate": 4.9722457257119144e-06,
+      "loss": 0.0057,
+      "num_tokens": 2472768.0,
+      "reward": 0.77099609375,
+      "reward_std": 0.011357970535755157,
+      "rewards//mean": 0.77099609375,
+      "rewards//std": 0.027263915166258812,
+      "step": 286
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0574,
+      "grad_norm": 0.8311311602592468,
+      "kl": 0.12657944671809673,
+      "learning_rate": 4.972009459250903e-06,
+      "loss": 0.0051,
+      "num_tokens": 2481432.0,
+      "reward": 0.72271728515625,
+      "reward_std": 0.012970637530088425,
+      "rewards//mean": 0.72271728515625,
+      "rewards//std": 0.03625509515404701,
+      "step": 287
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0576,
+      "grad_norm": 1.064893126487732,
+      "kl": 0.16098652640357614,
+      "learning_rate": 4.971772197064776e-06,
+      "loss": 0.0064,
+      "num_tokens": 2490120.0,
+      "reward": 0.73114013671875,
+      "reward_std": 0.013818023726344109,
+      "rewards//mean": 0.73114013671875,
+      "rewards//std": 0.03643544018268585,
+      "step": 288
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0578,
+      "grad_norm": 0.957671046257019,
+      "kl": 0.19215518794953823,
+      "learning_rate": 4.971533939249105e-06,
+      "loss": 0.0077,
+      "num_tokens": 2498832.0,
+      "reward": 0.73681640625,
+      "reward_std": 0.014772996306419373,
+      "rewards//mean": 0.73681640625,
+      "rewards//std": 0.04102522134780884,
+      "step": 289
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.058,
+      "grad_norm": 0.9647096395492554,
+      "kl": 0.1939010415226221,
+      "learning_rate": 4.9712946858998576e-06,
+      "loss": 0.0078,
+      "num_tokens": 2507544.0,
+      "reward": 0.75213623046875,
+      "reward_std": 0.012323443777859211,
+      "rewards//mean": 0.75213623046875,
+      "rewards//std": 0.036742422729730606,
+      "step": 290
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0582,
+      "grad_norm": 0.9463180899620056,
+      "kl": 0.1474277568049729,
+      "learning_rate": 4.971054437113406e-06,
+      "loss": 0.0059,
+      "num_tokens": 2516200.0,
+      "reward": 0.74615478515625,
+      "reward_std": 0.01541908085346222,
+      "rewards//mean": 0.74615478515625,
+      "rewards//std": 0.0504126213490963,
+      "step": 291
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0584,
+      "grad_norm": 1.2873408794403076,
+      "kl": 0.18494170205667615,
+      "learning_rate": 4.9708131929865235e-06,
+      "loss": 0.0074,
+      "num_tokens": 2524768.0,
+      "reward": 0.74810791015625,
+      "reward_std": 0.01592198945581913,
+      "rewards//mean": 0.74810791015625,
+      "rewards//std": 0.04342567175626755,
+      "step": 292
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0586,
+      "grad_norm": 0.9283332228660583,
+      "kl": 0.15322866081260145,
+      "learning_rate": 4.970570953616383e-06,
+      "loss": 0.0061,
+      "num_tokens": 2533440.0,
+      "reward": 0.74658203125,
+      "reward_std": 0.012621916830539703,
+      "rewards//mean": 0.74658203125,
+      "rewards//std": 0.03913993015885353,
+      "step": 293
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0588,
+      "grad_norm": 2.470165967941284,
+      "kl": 0.21047021076083183,
+      "learning_rate": 4.970327719100556e-06,
+      "loss": 0.0084,
+      "num_tokens": 2542128.0,
+      "reward": 0.7777099609375,
+      "reward_std": 0.013461882248520851,
+      "rewards//mean": 0.7777099609375,
+      "rewards//std": 0.0339735671877861,
+      "step": 294
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.059,
+      "grad_norm": 0.9362149238586426,
+      "kl": 0.14741351851262152,
+      "learning_rate": 4.970083489537021e-06,
+      "loss": 0.0059,
+      "num_tokens": 2550680.0,
+      "reward": 0.72869873046875,
+      "reward_std": 0.014110936783254147,
+      "rewards//mean": 0.72869873046875,
+      "rewards//std": 0.03162695840001106,
+      "step": 295
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0592,
+      "grad_norm": 0.8844790458679199,
+      "kl": 0.1842191582545638,
+      "learning_rate": 4.96983826502415e-06,
+      "loss": 0.0074,
+      "num_tokens": 2559424.0,
+      "reward": 0.7591552734375,
+      "reward_std": 0.008938804268836975,
+      "rewards//mean": 0.7591552734375,
+      "rewards//std": 0.04523535072803497,
+      "step": 296
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0594,
+      "grad_norm": 1.2818913459777832,
+      "kl": 0.27794970013201237,
+      "learning_rate": 4.969592045660723e-06,
+      "loss": 0.0111,
+      "num_tokens": 2568040.0,
+      "reward": 0.78643798828125,
+      "reward_std": 0.01272084191441536,
+      "rewards//mean": 0.78643798828125,
+      "rewards//std": 0.030282124876976013,
+      "step": 297
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0596,
+      "grad_norm": 1.2021187543869019,
+      "kl": 0.19154435116797686,
+      "learning_rate": 4.969344831545914e-06,
+      "loss": 0.0077,
+      "num_tokens": 2576776.0,
+      "reward": 0.75469970703125,
+      "reward_std": 0.01579802855849266,
+      "rewards//mean": 0.75469970703125,
+      "rewards//std": 0.040667545050382614,
+      "step": 298
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0598,
+      "grad_norm": 1.1095116138458252,
+      "kl": 0.2089026332832873,
+      "learning_rate": 4.969096622779303e-06,
+      "loss": 0.0084,
+      "num_tokens": 2585392.0,
+      "reward": 0.71600341796875,
+      "reward_std": 0.01274610310792923,
+      "rewards//mean": 0.71600341796875,
+      "rewards//std": 0.0431722030043602,
+      "step": 299
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.06,
+      "grad_norm": 1.059348702430725,
+      "kl": 0.20234669605270028,
+      "learning_rate": 4.968847419460867e-06,
+      "loss": 0.0081,
+      "num_tokens": 2594032.0,
+      "reward": 0.7236328125,
+      "reward_std": 0.013915905728936195,
+      "rewards//mean": 0.7236328125,
+      "rewards//std": 0.0312344953417778,
+      "step": 300
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0602,
+      "grad_norm": 1.2769887447357178,
+      "kl": 0.22755335364490747,
+      "learning_rate": 4.968597221690986e-06,
+      "loss": 0.0091,
+      "num_tokens": 2602736.0,
+      "reward": 0.72149658203125,
+      "reward_std": 0.011407001875340939,
+      "rewards//mean": 0.72149658203125,
+      "rewards//std": 0.049202706664800644,
+      "step": 301
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0604,
+      "grad_norm": 1.2020509243011475,
+      "kl": 0.24635399505496025,
+      "learning_rate": 4.96834602957044e-06,
+      "loss": 0.0099,
+      "num_tokens": 2611384.0,
+      "reward": 0.76763916015625,
+      "reward_std": 0.016048027202486992,
+      "rewards//mean": 0.76763916015625,
+      "rewards//std": 0.04085657000541687,
+      "step": 302
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0606,
+      "grad_norm": 1.436669111251831,
+      "kl": 0.25481397192925215,
+      "learning_rate": 4.968093843200407e-06,
+      "loss": 0.0102,
+      "num_tokens": 2620152.0,
+      "reward": 0.73895263671875,
+      "reward_std": 0.016838543117046356,
+      "rewards//mean": 0.73895263671875,
+      "rewards//std": 0.05139780789613724,
+      "step": 303
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0608,
+      "grad_norm": 1.3443458080291748,
+      "kl": 0.2639510128647089,
+      "learning_rate": 4.96784066268247e-06,
+      "loss": 0.0106,
+      "num_tokens": 2628816.0,
+      "reward": 0.745361328125,
+      "reward_std": 0.014135653153061867,
+      "rewards//mean": 0.745361328125,
+      "rewards//std": 0.03560379147529602,
+      "step": 304
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.061,
+      "grad_norm": 1.522937297821045,
+      "kl": 0.24147153925150633,
+      "learning_rate": 4.967586488118609e-06,
+      "loss": 0.0097,
+      "num_tokens": 2637496.0,
+      "reward": 0.7392578125,
+      "reward_std": 0.01406162977218628,
+      "rewards//mean": 0.7392578125,
+      "rewards//std": 0.028467724099755287,
+      "step": 305
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0612,
+      "grad_norm": 1.3553171157836914,
+      "kl": 0.24848425947129726,
+      "learning_rate": 4.967331319611206e-06,
+      "loss": 0.0099,
+      "num_tokens": 2646104.0,
+      "reward": 0.7559814453125,
+      "reward_std": 0.014312982559204102,
+      "rewards//mean": 0.7559814453125,
+      "rewards//std": 0.03511122614145279,
+      "step": 306
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0614,
+      "grad_norm": 1.5322372913360596,
+      "kl": 0.25969485752284527,
+      "learning_rate": 4.9670751572630425e-06,
+      "loss": 0.0104,
+      "num_tokens": 2654744.0,
+      "reward": 0.7608642578125,
+      "reward_std": 0.013267126865684986,
+      "rewards//mean": 0.7608642578125,
+      "rewards//std": 0.027817683294415474,
+      "step": 307
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0616,
+      "grad_norm": 1.4864404201507568,
+      "kl": 0.2588785719126463,
+      "learning_rate": 4.9668180011773e-06,
+      "loss": 0.0104,
+      "num_tokens": 2663376.0,
+      "reward": 0.71484375,
+      "reward_std": 0.01414964348077774,
+      "rewards//mean": 0.71484375,
+      "rewards//std": 0.02796124666929245,
+      "step": 308
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0618,
+      "grad_norm": 1.291443943977356,
+      "kl": 0.2382829338312149,
+      "learning_rate": 4.966559851457562e-06,
+      "loss": 0.0095,
+      "num_tokens": 2671952.0,
+      "reward": 0.7293701171875,
+      "reward_std": 0.015213550999760628,
+      "rewards//mean": 0.7293701171875,
+      "rewards//std": 0.03245498239994049,
+      "step": 309
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.062,
+      "grad_norm": 1.3344167470932007,
+      "kl": 0.2496517887338996,
+      "learning_rate": 4.966300708207811e-06,
+      "loss": 0.01,
+      "num_tokens": 2680624.0,
+      "reward": 0.73394775390625,
+      "reward_std": 0.01914324052631855,
+      "rewards//mean": 0.73394775390625,
+      "rewards//std": 0.04364334046840668,
+      "step": 310
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0622,
+      "grad_norm": 1.3081969022750854,
+      "kl": 0.30021103471517563,
+      "learning_rate": 4.96604057153243e-06,
+      "loss": 0.012,
+      "num_tokens": 2689176.0,
+      "reward": 0.764892578125,
+      "reward_std": 0.01673274114727974,
+      "rewards//mean": 0.764892578125,
+      "rewards//std": 0.040132150053977966,
+      "step": 311
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0624,
+      "grad_norm": 1.3401750326156616,
+      "kl": 0.2593008913099766,
+      "learning_rate": 4.965779441536202e-06,
+      "loss": 0.0104,
+      "num_tokens": 2697864.0,
+      "reward": 0.74420166015625,
+      "reward_std": 0.01244452502578497,
+      "rewards//mean": 0.74420166015625,
+      "rewards//std": 0.03462515026330948,
+      "step": 312
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0626,
+      "grad_norm": 1.4522110223770142,
+      "kl": 0.3523233197629452,
+      "learning_rate": 4.965517318324308e-06,
+      "loss": 0.0141,
+      "num_tokens": 2706480.0,
+      "reward": 0.71630859375,
+      "reward_std": 0.011550749652087688,
+      "rewards//mean": 0.71630859375,
+      "rewards//std": 0.026689305901527405,
+      "step": 313
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0628,
+      "grad_norm": 1.3593254089355469,
+      "kl": 0.3362383022904396,
+      "learning_rate": 4.965254202002334e-06,
+      "loss": 0.0134,
+      "num_tokens": 2715056.0,
+      "reward": 0.75225830078125,
+      "reward_std": 0.015795797109603882,
+      "rewards//mean": 0.75225830078125,
+      "rewards//std": 0.0387437678873539,
+      "step": 314
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.063,
+      "grad_norm": 1.2711306810379028,
+      "kl": 0.26307030860334635,
+      "learning_rate": 4.964990092676263e-06,
+      "loss": 0.0105,
+      "num_tokens": 2723632.0,
+      "reward": 0.74615478515625,
+      "reward_std": 0.011496458202600479,
+      "rewards//mean": 0.74615478515625,
+      "rewards//std": 0.038194045424461365,
+      "step": 315
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0632,
+      "grad_norm": 1.2173610925674438,
+      "kl": 0.272955933585763,
+      "learning_rate": 4.964724990452476e-06,
+      "loss": 0.0109,
+      "num_tokens": 2732264.0,
+      "reward": 0.72149658203125,
+      "reward_std": 0.012493669055402279,
+      "rewards//mean": 0.72149658203125,
+      "rewards//std": 0.036469489336013794,
+      "step": 316
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0634,
+      "grad_norm": 1.143746256828308,
+      "kl": 0.24079665448516607,
+      "learning_rate": 4.9644588954377595e-06,
+      "loss": 0.0096,
+      "num_tokens": 2740960.0,
+      "reward": 0.7584228515625,
+      "reward_std": 0.015307648107409477,
+      "rewards//mean": 0.7584228515625,
+      "rewards//std": 0.03647971153259277,
+      "step": 317
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0636,
+      "grad_norm": 1.21216881275177,
+      "kl": 0.25471873860806227,
+      "learning_rate": 4.964191807739293e-06,
+      "loss": 0.0102,
+      "num_tokens": 2749528.0,
+      "reward": 0.741943359375,
+      "reward_std": 0.011934969574213028,
+      "rewards//mean": 0.741943359375,
+      "rewards//std": 0.02646489255130291,
+      "step": 318
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0638,
+      "grad_norm": 1.5371971130371094,
+      "kl": 0.3100880701094866,
+      "learning_rate": 4.963923727464661e-06,
+      "loss": 0.0124,
+      "num_tokens": 2758176.0,
+      "reward": 0.7532958984375,
+      "reward_std": 0.011484376154839993,
+      "rewards//mean": 0.7532958984375,
+      "rewards//std": 0.031002702191472054,
+      "step": 319
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.064,
+      "grad_norm": 1.2735233306884766,
+      "kl": 0.2549311425536871,
+      "learning_rate": 4.963654654721848e-06,
+      "loss": 0.0102,
+      "num_tokens": 2766760.0,
+      "reward": 0.7633056640625,
+      "reward_std": 0.01067253015935421,
+      "rewards//mean": 0.7633056640625,
+      "rewards//std": 0.03851751983165741,
+      "step": 320
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0642,
+      "grad_norm": 1.3543078899383545,
+      "kl": 0.33309008402284235,
+      "learning_rate": 4.963384589619233e-06,
+      "loss": 0.0133,
+      "num_tokens": 2775344.0,
+      "reward": 0.75543212890625,
+      "reward_std": 0.015507234260439873,
+      "rewards//mean": 0.75543212890625,
+      "rewards//std": 0.03842874616384506,
+      "step": 321
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0644,
+      "grad_norm": 1.2867757081985474,
+      "kl": 0.35989778488874435,
+      "learning_rate": 4.9631135322656e-06,
+      "loss": 0.0144,
+      "num_tokens": 2783880.0,
+      "reward": 0.75213623046875,
+      "reward_std": 0.013345220126211643,
+      "rewards//mean": 0.75213623046875,
+      "rewards//std": 0.0316346138715744,
+      "step": 322
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0646,
+      "grad_norm": 1.3027052879333496,
+      "kl": 0.3117766585201025,
+      "learning_rate": 4.962841482770131e-06,
+      "loss": 0.0125,
+      "num_tokens": 2792480.0,
+      "reward": 0.75042724609375,
+      "reward_std": 0.021965457126498222,
+      "rewards//mean": 0.75042724609375,
+      "rewards//std": 0.03844450041651726,
+      "step": 323
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0648,
+      "grad_norm": 1.2578948736190796,
+      "kl": 0.31469903141260147,
+      "learning_rate": 4.962568441242408e-06,
+      "loss": 0.0126,
+      "num_tokens": 2801088.0,
+      "reward": 0.723388671875,
+      "reward_std": 0.015950549393892288,
+      "rewards//mean": 0.723388671875,
+      "rewards//std": 0.03848705068230629,
+      "step": 324
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.065,
+      "grad_norm": 1.3456346988677979,
+      "kl": 0.26953551825135946,
+      "learning_rate": 4.962294407792411e-06,
+      "loss": 0.0108,
+      "num_tokens": 2809768.0,
+      "reward": 0.747802734375,
+      "reward_std": 0.015800267457962036,
+      "rewards//mean": 0.747802734375,
+      "rewards//std": 0.03215061128139496,
+      "step": 325
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0652,
+      "grad_norm": 1.6204522848129272,
+      "kl": 0.34877745993435383,
+      "learning_rate": 4.962019382530521e-06,
+      "loss": 0.014,
+      "num_tokens": 2818328.0,
+      "reward": 0.756591796875,
+      "reward_std": 0.017749017104506493,
+      "rewards//mean": 0.756591796875,
+      "rewards//std": 0.03980492055416107,
+      "step": 326
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0654,
+      "grad_norm": 1598.2496337890625,
+      "kl": 9.96565246488899,
+      "learning_rate": 4.961743365567517e-06,
+      "loss": 0.3986,
+      "num_tokens": 2826984.0,
+      "reward": 0.71923828125,
+      "reward_std": 0.014437740668654442,
+      "rewards//mean": 0.71923828125,
+      "rewards//std": 0.05055179446935654,
+      "step": 327
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0656,
+      "grad_norm": 1.0102441310882568,
+      "kl": 0.30338819324970245,
+      "learning_rate": 4.961466357014581e-06,
+      "loss": 0.0121,
+      "num_tokens": 2835544.0,
+      "reward": 0.75927734375,
+      "reward_std": 0.011439365334808826,
+      "rewards//mean": 0.75927734375,
+      "rewards//std": 0.02604631707072258,
+      "step": 328
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0658,
+      "grad_norm": 451.71466064453125,
+      "kl": 1.1315573640167713,
+      "learning_rate": 4.961188356983291e-06,
+      "loss": 0.0453,
+      "num_tokens": 2844184.0,
+      "reward": 0.77728271484375,
+      "reward_std": 0.014329792931675911,
+      "rewards//mean": 0.77728271484375,
+      "rewards//std": 0.030076975002884865,
+      "step": 329
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.066,
+      "grad_norm": 1.7841613292694092,
+      "kl": 0.32306745275855064,
+      "learning_rate": 4.960909365585624e-06,
+      "loss": 0.0129,
+      "num_tokens": 2852824.0,
+      "reward": 0.7259521484375,
+      "reward_std": 0.012303611263632774,
+      "rewards//mean": 0.7259521484375,
+      "rewards//std": 0.024766186252236366,
+      "step": 330
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0662,
+      "grad_norm": 1.2208861112594604,
+      "kl": 0.35031407698988914,
+      "learning_rate": 4.960629382933959e-06,
+      "loss": 0.014,
+      "num_tokens": 2861568.0,
+      "reward": 0.75592041015625,
+      "reward_std": 0.013036654330790043,
+      "rewards//mean": 0.75592041015625,
+      "rewards//std": 0.028763895854353905,
+      "step": 331
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0664,
+      "grad_norm": 45.699703216552734,
+      "kl": 0.37541868537664413,
+      "learning_rate": 4.960348409141074e-06,
+      "loss": 0.015,
+      "num_tokens": 2870168.0,
+      "reward": 0.76629638671875,
+      "reward_std": 0.014367573894560337,
+      "rewards//mean": 0.76629638671875,
+      "rewards//std": 0.03430584445595741,
+      "step": 332
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0666,
+      "grad_norm": 1.0523972511291504,
+      "kl": 0.2711593806743622,
+      "learning_rate": 4.960066444320143e-06,
+      "loss": 0.0108,
+      "num_tokens": 2878752.0,
+      "reward": 0.72601318359375,
+      "reward_std": 0.016535844653844833,
+      "rewards//mean": 0.72601318359375,
+      "rewards//std": 0.03955461084842682,
+      "step": 333
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0668,
+      "grad_norm": 1.1287596225738525,
+      "kl": 0.27480507269501686,
+      "learning_rate": 4.959783488584743e-06,
+      "loss": 0.011,
+      "num_tokens": 2887504.0,
+      "reward": 0.7252197265625,
+      "reward_std": 0.013544876128435135,
+      "rewards//mean": 0.7252197265625,
+      "rewards//std": 0.036889057606458664,
+      "step": 334
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.067,
+      "grad_norm": 1.9436277151107788,
+      "kl": 0.26933558098971844,
+      "learning_rate": 4.9594995420488475e-06,
+      "loss": 0.0108,
+      "num_tokens": 2896176.0,
+      "reward": 0.70745849609375,
+      "reward_std": 0.01362568698823452,
+      "rewards//mean": 0.70745849609375,
+      "rewards//std": 0.0338621586561203,
+      "step": 335
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0672,
+      "grad_norm": 1.3394960165023804,
+      "kl": 0.2702603470534086,
+      "learning_rate": 4.959214604826831e-06,
+      "loss": 0.0108,
+      "num_tokens": 2904808.0,
+      "reward": 0.73876953125,
+      "reward_std": 0.015888595953583717,
+      "rewards//mean": 0.73876953125,
+      "rewards//std": 0.03880433365702629,
+      "step": 336
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0674,
+      "grad_norm": 1.188582181930542,
+      "kl": 0.3325184481218457,
+      "learning_rate": 4.958928677033465e-06,
+      "loss": 0.0133,
+      "num_tokens": 2913544.0,
+      "reward": 0.73760986328125,
+      "reward_std": 0.01286403276026249,
+      "rewards//mean": 0.73760986328125,
+      "rewards//std": 0.03628222644329071,
+      "step": 337
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0676,
+      "grad_norm": 1.0829931497573853,
+      "kl": 0.2599369240924716,
+      "learning_rate": 4.9586417587839225e-06,
+      "loss": 0.0104,
+      "num_tokens": 2922208.0,
+      "reward": 0.74237060546875,
+      "reward_std": 0.009288130328059196,
+      "rewards//mean": 0.74237060546875,
+      "rewards//std": 0.0271921269595623,
+      "step": 338
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0678,
+      "grad_norm": 1.090649127960205,
+      "kl": 0.35008446872234344,
+      "learning_rate": 4.958353850193773e-06,
+      "loss": 0.014,
+      "num_tokens": 2930904.0,
+      "reward": 0.7298583984375,
+      "reward_std": 0.009747210890054703,
+      "rewards//mean": 0.7298583984375,
+      "rewards//std": 0.031018322333693504,
+      "step": 339
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.068,
+      "grad_norm": 1.0257357358932495,
+      "kl": 0.2736462540924549,
+      "learning_rate": 4.958064951378988e-06,
+      "loss": 0.0109,
+      "num_tokens": 2939600.0,
+      "reward": 0.74664306640625,
+      "reward_std": 0.013303879648447037,
+      "rewards//mean": 0.74664306640625,
+      "rewards//std": 0.026327185332775116,
+      "step": 340
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0682,
+      "grad_norm": 1.8254125118255615,
+      "kl": 0.3379829414188862,
+      "learning_rate": 4.957775062455933e-06,
+      "loss": 0.0135,
+      "num_tokens": 2948432.0,
+      "reward": 0.74041748046875,
+      "reward_std": 0.011445442214608192,
+      "rewards//mean": 0.74041748046875,
+      "rewards//std": 0.023927679285407066,
+      "step": 341
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0684,
+      "grad_norm": 1.151408314704895,
+      "kl": 0.3058789037168026,
+      "learning_rate": 4.957484183541378e-06,
+      "loss": 0.0122,
+      "num_tokens": 2957032.0,
+      "reward": 0.775390625,
+      "reward_std": 0.010520260781049728,
+      "rewards//mean": 0.775390625,
+      "rewards//std": 0.020039275288581848,
+      "step": 342
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0686,
+      "grad_norm": 3.0106112957000732,
+      "kl": 0.330666683614254,
+      "learning_rate": 4.957192314752487e-06,
+      "loss": 0.0132,
+      "num_tokens": 2965680.0,
+      "reward": 0.7501220703125,
+      "reward_std": 0.0123628880828619,
+      "rewards//mean": 0.7501220703125,
+      "rewards//std": 0.03424517437815666,
+      "step": 343
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0688,
+      "grad_norm": 9.348411560058594,
+      "kl": 0.3467661701142788,
+      "learning_rate": 4.9568994562068265e-06,
+      "loss": 0.0139,
+      "num_tokens": 2974304.0,
+      "reward": 0.7127685546875,
+      "reward_std": 0.013961128890514374,
+      "rewards//mean": 0.7127685546875,
+      "rewards//std": 0.038146305829286575,
+      "step": 344
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.069,
+      "grad_norm": 1.3383392095565796,
+      "kl": 0.23608121927827597,
+      "learning_rate": 4.9566056080223576e-06,
+      "loss": 0.0094,
+      "num_tokens": 2982920.0,
+      "reward": 0.7254638671875,
+      "reward_std": 0.019042517989873886,
+      "rewards//mean": 0.7254638671875,
+      "rewards//std": 0.05562053620815277,
+      "step": 345
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0692,
+      "grad_norm": 1.1454135179519653,
+      "kl": 0.3010788504034281,
+      "learning_rate": 4.9563107703174444e-06,
+      "loss": 0.012,
+      "num_tokens": 2991640.0,
+      "reward": 0.76824951171875,
+      "reward_std": 0.013647787272930145,
+      "rewards//mean": 0.76824951171875,
+      "rewards//std": 0.02736636996269226,
+      "step": 346
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0694,
+      "grad_norm": 1.3544073104858398,
+      "kl": 0.34564877301454544,
+      "learning_rate": 4.956014943210845e-06,
+      "loss": 0.0138,
+      "num_tokens": 3000224.0,
+      "reward": 0.7606201171875,
+      "reward_std": 0.011027848348021507,
+      "rewards//mean": 0.7606201171875,
+      "rewards//std": 0.029086079448461533,
+      "step": 347
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0696,
+      "grad_norm": 1.1409554481506348,
+      "kl": 0.29469297640025616,
+      "learning_rate": 4.9557181268217225e-06,
+      "loss": 0.0118,
+      "num_tokens": 3008880.0,
+      "reward": 0.73162841796875,
+      "reward_std": 0.017540261149406433,
+      "rewards//mean": 0.73162841796875,
+      "rewards//std": 0.03117768093943596,
+      "step": 348
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0698,
+      "grad_norm": 1.174091100692749,
+      "kl": 0.34364804439246655,
+      "learning_rate": 4.9554203212696304e-06,
+      "loss": 0.0137,
+      "num_tokens": 3017536.0,
+      "reward": 0.73602294921875,
+      "reward_std": 0.0178272295743227,
+      "rewards//mean": 0.73602294921875,
+      "rewards//std": 0.038189683109521866,
+      "step": 349
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.07,
+      "grad_norm": 1.0863193273544312,
+      "kl": 0.2888754541054368,
+      "learning_rate": 4.955121526674528e-06,
+      "loss": 0.0116,
+      "num_tokens": 3026160.0,
+      "reward": 0.76239013671875,
+      "reward_std": 0.015270760282874107,
+      "rewards//mean": 0.76239013671875,
+      "rewards//std": 0.04026879370212555,
+      "step": 350
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0702,
+      "grad_norm": 1.3240951299667358,
+      "kl": 0.32703710440546274,
+      "learning_rate": 4.9548217431567665e-06,
+      "loss": 0.0131,
+      "num_tokens": 3034792.0,
+      "reward": 0.77569580078125,
+      "reward_std": 0.013240109197795391,
+      "rewards//mean": 0.77569580078125,
+      "rewards//std": 0.0282268188893795,
+      "step": 351
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0704,
+      "grad_norm": 1.2549176216125488,
+      "kl": 0.27426617220044136,
+      "learning_rate": 4.9545209708371025e-06,
+      "loss": 0.011,
+      "num_tokens": 3043432.0,
+      "reward": 0.75177001953125,
+      "reward_std": 0.009705094620585442,
+      "rewards//mean": 0.75177001953125,
+      "rewards//std": 0.0316346138715744,
+      "step": 352
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0706,
+      "grad_norm": 1.207445502281189,
+      "kl": 0.3094801548868418,
+      "learning_rate": 4.9542192098366835e-06,
+      "loss": 0.0124,
+      "num_tokens": 3052008.0,
+      "reward": 0.76214599609375,
+      "reward_std": 0.015533574856817722,
+      "rewards//mean": 0.76214599609375,
+      "rewards//std": 0.029373183846473694,
+      "step": 353
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0708,
+      "grad_norm": 0.9336227774620056,
+      "kl": 0.3114245068281889,
+      "learning_rate": 4.95391646027706e-06,
+      "loss": 0.0125,
+      "num_tokens": 3060680.0,
+      "reward": 0.7603759765625,
+      "reward_std": 0.013422933407127857,
+      "rewards//mean": 0.7603759765625,
+      "rewards//std": 0.034570734947919846,
+      "step": 354
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.071,
+      "grad_norm": 0.9703741073608398,
+      "kl": 0.3360181748867035,
+      "learning_rate": 4.953612722280181e-06,
+      "loss": 0.0134,
+      "num_tokens": 3069312.0,
+      "reward": 0.7603759765625,
+      "reward_std": 0.014663382433354855,
+      "rewards//mean": 0.7603759765625,
+      "rewards//std": 0.032032448798418045,
+      "step": 355
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0712,
+      "grad_norm": 0.9895470142364502,
+      "kl": 0.32697275839746,
+      "learning_rate": 4.953307995968391e-06,
+      "loss": 0.0131,
+      "num_tokens": 3078032.0,
+      "reward": 0.74468994140625,
+      "reward_std": 0.010848162695765495,
+      "rewards//mean": 0.74468994140625,
+      "rewards//std": 0.025433441624045372,
+      "step": 356
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0714,
+      "grad_norm": 1.0759690999984741,
+      "kl": 0.3138846158981323,
+      "learning_rate": 4.953002281464432e-06,
+      "loss": 0.0126,
+      "num_tokens": 3086656.0,
+      "reward": 0.738037109375,
+      "reward_std": 0.011200450360774994,
+      "rewards//mean": 0.738037109375,
+      "rewards//std": 0.019008060917258263,
+      "step": 357
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0716,
+      "grad_norm": 1.1179635524749756,
+      "kl": 0.2949511222541332,
+      "learning_rate": 4.952695578891449e-06,
+      "loss": 0.0118,
+      "num_tokens": 3095184.0,
+      "reward": 0.74249267578125,
+      "reward_std": 0.010483279824256897,
+      "rewards//mean": 0.74249267578125,
+      "rewards//std": 0.034091606736183167,
+      "step": 358
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0718,
+      "grad_norm": 1.4193079471588135,
+      "kl": 0.3405211642384529,
+      "learning_rate": 4.9523878883729794e-06,
+      "loss": 0.0136,
+      "num_tokens": 3103768.0,
+      "reward": 0.78057861328125,
+      "reward_std": 0.01512511633336544,
+      "rewards//mean": 0.78057861328125,
+      "rewards//std": 0.028378715738654137,
+      "step": 359
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.072,
+      "grad_norm": 1.239197015762329,
+      "kl": 0.34275365993380547,
+      "learning_rate": 4.952079210032962e-06,
+      "loss": 0.0137,
+      "num_tokens": 3112464.0,
+      "reward": 0.75830078125,
+      "reward_std": 0.00690212519839406,
+      "rewards//mean": 0.75830078125,
+      "rewards//std": 0.029177792370319366,
+      "step": 360
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0722,
+      "grad_norm": 1.2645068168640137,
+      "kl": 0.22989958012476563,
+      "learning_rate": 4.951769543995731e-06,
+      "loss": 0.0092,
+      "num_tokens": 3121000.0,
+      "reward": 0.7113037109375,
+      "reward_std": 0.014688584953546524,
+      "rewards//mean": 0.7113037109375,
+      "rewards//std": 0.043239694088697433,
+      "step": 361
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0724,
+      "grad_norm": 0.9607939720153809,
+      "kl": 0.2874492518603802,
+      "learning_rate": 4.951458890386021e-06,
+      "loss": 0.0115,
+      "num_tokens": 3129608.0,
+      "reward": 0.77667236328125,
+      "reward_std": 0.019359227269887924,
+      "rewards//mean": 0.77667236328125,
+      "rewards//std": 0.03485913202166557,
+      "step": 362
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0726,
+      "grad_norm": 1.1458990573883057,
+      "kl": 0.3057211823761463,
+      "learning_rate": 4.951147249328964e-06,
+      "loss": 0.0122,
+      "num_tokens": 3138360.0,
+      "reward": 0.74261474609375,
+      "reward_std": 0.016560913994908333,
+      "rewards//mean": 0.74261474609375,
+      "rewards//std": 0.045068517327308655,
+      "step": 363
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0728,
+      "grad_norm": 1.1187580823898315,
+      "kl": 0.3212597221136093,
+      "learning_rate": 4.950834620950089e-06,
+      "loss": 0.0129,
+      "num_tokens": 3146912.0,
+      "reward": 0.7310791015625,
+      "reward_std": 0.015014410018920898,
+      "rewards//mean": 0.7310791015625,
+      "rewards//std": 0.04567229375243187,
+      "step": 364
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.073,
+      "grad_norm": 1.4113508462905884,
+      "kl": 0.35538918152451515,
+      "learning_rate": 4.9505210053753204e-06,
+      "loss": 0.0142,
+      "num_tokens": 3155568.0,
+      "reward": 0.7164306640625,
+      "reward_std": 0.013062847778201103,
+      "rewards//mean": 0.7164306640625,
+      "rewards//std": 0.049687836319208145,
+      "step": 365
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0732,
+      "grad_norm": 1.2345703840255737,
+      "kl": 0.288858812302351,
+      "learning_rate": 4.950206402730984e-06,
+      "loss": 0.0116,
+      "num_tokens": 3164336.0,
+      "reward": 0.76116943359375,
+      "reward_std": 0.013638054020702839,
+      "rewards//mean": 0.76116943359375,
+      "rewards//std": 0.04553765431046486,
+      "step": 366
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0734,
+      "grad_norm": 1.2489417791366577,
+      "kl": 0.3488999232649803,
+      "learning_rate": 4.949890813143802e-06,
+      "loss": 0.014,
+      "num_tokens": 3173144.0,
+      "reward": 0.7589111328125,
+      "reward_std": 0.010227528400719166,
+      "rewards//mean": 0.7589111328125,
+      "rewards//std": 0.03326575458049774,
+      "step": 367
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0736,
+      "grad_norm": 1.1428223848342896,
+      "kl": 0.33989420160651207,
+      "learning_rate": 4.949574236740893e-06,
+      "loss": 0.0136,
+      "num_tokens": 3181768.0,
+      "reward": 0.73931884765625,
+      "reward_std": 0.008424145169556141,
+      "rewards//mean": 0.73931884765625,
+      "rewards//std": 0.02814464643597603,
+      "step": 368
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0738,
+      "grad_norm": 1.1611658334732056,
+      "kl": 0.32859067991375923,
+      "learning_rate": 4.949256673649774e-06,
+      "loss": 0.0131,
+      "num_tokens": 3190400.0,
+      "reward": 0.71636962890625,
+      "reward_std": 0.011011059395968914,
+      "rewards//mean": 0.71636962890625,
+      "rewards//std": 0.041278988122940063,
+      "step": 369
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.074,
+      "grad_norm": 1.2848570346832275,
+      "kl": 0.35491205751895905,
+      "learning_rate": 4.94893812399836e-06,
+      "loss": 0.0142,
+      "num_tokens": 3199032.0,
+      "reward": 0.77215576171875,
+      "reward_std": 0.01237148605287075,
+      "rewards//mean": 0.77215576171875,
+      "rewards//std": 0.03036997839808464,
+      "step": 370
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0742,
+      "grad_norm": 1.300308346748352,
+      "kl": 0.3731778897345066,
+      "learning_rate": 4.948618587914963e-06,
+      "loss": 0.0149,
+      "num_tokens": 3207600.0,
+      "reward": 0.7557373046875,
+      "reward_std": 0.01102093979716301,
+      "rewards//mean": 0.7557373046875,
+      "rewards//std": 0.025770245119929314,
+      "step": 371
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0744,
+      "grad_norm": 1.1986355781555176,
+      "kl": 0.33789339661598206,
+      "learning_rate": 4.948298065528292e-06,
+      "loss": 0.0135,
+      "num_tokens": 3216240.0,
+      "reward": 0.74041748046875,
+      "reward_std": 0.012240855023264885,
+      "rewards//mean": 0.74041748046875,
+      "rewards//std": 0.03901242837309837,
+      "step": 372
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0746,
+      "grad_norm": 1.1584309339523315,
+      "kl": 0.3138528410345316,
+      "learning_rate": 4.947976556967452e-06,
+      "loss": 0.0126,
+      "num_tokens": 3224856.0,
+      "reward": 0.790283203125,
+      "reward_std": 0.010637026280164719,
+      "rewards//mean": 0.790283203125,
+      "rewards//std": 0.028278857469558716,
+      "step": 373
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0748,
+      "grad_norm": 1.1529393196105957,
+      "kl": 0.32554432936012745,
+      "learning_rate": 4.947654062361949e-06,
+      "loss": 0.013,
+      "num_tokens": 3233608.0,
+      "reward": 0.7681884765625,
+      "reward_std": 0.016204483807086945,
+      "rewards//mean": 0.7681884765625,
+      "rewards//std": 0.03761722892522812,
+      "step": 374
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.075,
+      "grad_norm": 1.137768030166626,
+      "kl": 0.3275693580508232,
+      "learning_rate": 4.9473305818416805e-06,
+      "loss": 0.0131,
+      "num_tokens": 3242224.0,
+      "reward": 0.72662353515625,
+      "reward_std": 0.012011650949716568,
+      "rewards//mean": 0.72662353515625,
+      "rewards//std": 0.04576380178332329,
+      "step": 375
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0752,
+      "grad_norm": 1.1497950553894043,
+      "kl": 0.373350128531456,
+      "learning_rate": 4.947006115536947e-06,
+      "loss": 0.0149,
+      "num_tokens": 3250880.0,
+      "reward": 0.7626953125,
+      "reward_std": 0.0052624596282839775,
+      "rewards//mean": 0.7626953125,
+      "rewards//std": 0.034618210047483444,
+      "step": 376
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0754,
+      "grad_norm": 1.2146048545837402,
+      "kl": 0.31162706576287746,
+      "learning_rate": 4.946680663578443e-06,
+      "loss": 0.0125,
+      "num_tokens": 3259592.0,
+      "reward": 0.7554931640625,
+      "reward_std": 0.01137151475995779,
+      "rewards//mean": 0.7554931640625,
+      "rewards//std": 0.03868066519498825,
+      "step": 377
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0756,
+      "grad_norm": 1.0558278560638428,
+      "kl": 0.34958504140377045,
+      "learning_rate": 4.946354226097261e-06,
+      "loss": 0.014,
+      "num_tokens": 3268216.0,
+      "reward": 0.75439453125,
+      "reward_std": 0.01355811208486557,
+      "rewards//mean": 0.75439453125,
+      "rewards//std": 0.04268030822277069,
+      "step": 378
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0758,
+      "grad_norm": 1.3659820556640625,
+      "kl": 0.3736311122775078,
+      "learning_rate": 4.946026803224888e-06,
+      "loss": 0.0149,
+      "num_tokens": 3276832.0,
+      "reward": 0.77215576171875,
+      "reward_std": 0.00643074419349432,
+      "rewards//mean": 0.77215576171875,
+      "rewards//std": 0.028365911915898323,
+      "step": 379
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.076,
+      "grad_norm": 1.0803120136260986,
+      "kl": 0.3101845942437649,
+      "learning_rate": 4.945698395093212e-06,
+      "loss": 0.0124,
+      "num_tokens": 3285440.0,
+      "reward": 0.7320556640625,
+      "reward_std": 0.012730730697512627,
+      "rewards//mean": 0.7320556640625,
+      "rewards//std": 0.04066479951143265,
+      "step": 380
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0762,
+      "grad_norm": 1.184386134147644,
+      "kl": 0.33812105655670166,
+      "learning_rate": 4.9453690018345144e-06,
+      "loss": 0.0135,
+      "num_tokens": 3294056.0,
+      "reward": 0.765869140625,
+      "reward_std": 0.010241934098303318,
+      "rewards//mean": 0.765869140625,
+      "rewards//std": 0.03962195664644241,
+      "step": 381
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0764,
+      "grad_norm": 1.0949220657348633,
+      "kl": 0.32981303334236145,
+      "learning_rate": 4.9450386235814755e-06,
+      "loss": 0.0132,
+      "num_tokens": 3302832.0,
+      "reward": 0.74432373046875,
+      "reward_std": 0.011420607566833496,
+      "rewards//mean": 0.74432373046875,
+      "rewards//std": 0.036144278943538666,
+      "step": 382
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0766,
+      "grad_norm": 0.9847798943519592,
+      "kl": 0.2912776917219162,
+      "learning_rate": 4.944707260467172e-06,
+      "loss": 0.0117,
+      "num_tokens": 3311488.0,
+      "reward": 0.751953125,
+      "reward_std": 0.012202553451061249,
+      "rewards//mean": 0.751953125,
+      "rewards//std": 0.03254092112183571,
+      "step": 383
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0768,
+      "grad_norm": 1.0881670713424683,
+      "kl": 0.3519246131181717,
+      "learning_rate": 4.944374912625076e-06,
+      "loss": 0.0141,
+      "num_tokens": 3320128.0,
+      "reward": 0.721435546875,
+      "reward_std": 0.010593841783702374,
+      "rewards//mean": 0.721435546875,
+      "rewards//std": 0.04617677628993988,
+      "step": 384
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.077,
+      "grad_norm": 0.9181442856788635,
+      "kl": 0.348065834492445,
+      "learning_rate": 4.944041580189057e-06,
+      "loss": 0.0139,
+      "num_tokens": 3328744.0,
+      "reward": 0.7432861328125,
+      "reward_std": 0.009224426001310349,
+      "rewards//mean": 0.7432861328125,
+      "rewards//std": 0.026073908433318138,
+      "step": 385
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0772,
+      "grad_norm": 0.9651432037353516,
+      "kl": 0.2787171872332692,
+      "learning_rate": 4.943707263293382e-06,
+      "loss": 0.0111,
+      "num_tokens": 3337352.0,
+      "reward": 0.741943359375,
+      "reward_std": 0.014116060920059681,
+      "rewards//mean": 0.741943359375,
+      "rewards//std": 0.03688023239374161,
+      "step": 386
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0774,
+      "grad_norm": 1.0564086437225342,
+      "kl": 0.33349500969052315,
+      "learning_rate": 4.943371962072714e-06,
+      "loss": 0.0133,
+      "num_tokens": 3345952.0,
+      "reward": 0.779052734375,
+      "reward_std": 0.009006861597299576,
+      "rewards//mean": 0.779052734375,
+      "rewards//std": 0.03128775954246521,
+      "step": 387
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0776,
+      "grad_norm": 0.905786395072937,
+      "kl": 0.3205620348453522,
+      "learning_rate": 4.9430356766621114e-06,
+      "loss": 0.0128,
+      "num_tokens": 3354552.0,
+      "reward": 0.7352294921875,
+      "reward_std": 0.00872873142361641,
+      "rewards//mean": 0.7352294921875,
+      "rewards//std": 0.030068732798099518,
+      "step": 388
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0778,
+      "grad_norm": 1.0599631071090698,
+      "kl": 0.3847878910601139,
+      "learning_rate": 4.942698407197031e-06,
+      "loss": 0.0154,
+      "num_tokens": 3363192.0,
+      "reward": 0.741943359375,
+      "reward_std": 0.008737495169043541,
+      "rewards//mean": 0.741943359375,
+      "rewards//std": 0.030614785850048065,
+      "step": 389
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.078,
+      "grad_norm": 0.9868512749671936,
+      "kl": 0.32742261700332165,
+      "learning_rate": 4.942360153813324e-06,
+      "loss": 0.0131,
+      "num_tokens": 3371832.0,
+      "reward": 0.74371337890625,
+      "reward_std": 0.007218184880912304,
+      "rewards//mean": 0.74371337890625,
+      "rewards//std": 0.035629238933324814,
+      "step": 390
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0782,
+      "grad_norm": 1.0115092992782593,
+      "kl": 0.3397269584238529,
+      "learning_rate": 4.9420209166472386e-06,
+      "loss": 0.0136,
+      "num_tokens": 3380448.0,
+      "reward": 0.770751953125,
+      "reward_std": 0.015268933959305286,
+      "rewards//mean": 0.770751953125,
+      "rewards//std": 0.03136507794260979,
+      "step": 391
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0784,
+      "grad_norm": 0.9326263666152954,
+      "kl": 0.32895535230636597,
+      "learning_rate": 4.9416806958354206e-06,
+      "loss": 0.0132,
+      "num_tokens": 3389040.0,
+      "reward": 0.755126953125,
+      "reward_std": 0.013349458575248718,
+      "rewards//mean": 0.755126953125,
+      "rewards//std": 0.031357355415821075,
+      "step": 392
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0786,
+      "grad_norm": 0.7784397602081299,
+      "kl": 0.3131994195282459,
+      "learning_rate": 4.9413394915149094e-06,
+      "loss": 0.0125,
+      "num_tokens": 3397736.0,
+      "reward": 0.748779296875,
+      "reward_std": 0.011831846088171005,
+      "rewards//mean": 0.748779296875,
+      "rewards//std": 0.03131871297955513,
+      "step": 393
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0788,
+      "grad_norm": 0.9137967824935913,
+      "kl": 0.3138259369879961,
+      "learning_rate": 4.940997303823144e-06,
+      "loss": 0.0126,
+      "num_tokens": 3406328.0,
+      "reward": 0.7393798828125,
+      "reward_std": 0.009359323419630527,
+      "rewards//mean": 0.7393798828125,
+      "rewards//std": 0.01415227074176073,
+      "step": 394
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.079,
+      "grad_norm": 0.8206107020378113,
+      "kl": 0.2667227676138282,
+      "learning_rate": 4.940654132897957e-06,
+      "loss": 0.0107,
+      "num_tokens": 3414960.0,
+      "reward": 0.7227783203125,
+      "reward_std": 0.015779396519064903,
+      "rewards//mean": 0.7227783203125,
+      "rewards//std": 0.04207717627286911,
+      "step": 395
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0792,
+      "grad_norm": 1.1698734760284424,
+      "kl": 0.3585771173238754,
+      "learning_rate": 4.940309978877576e-06,
+      "loss": 0.0143,
+      "num_tokens": 3423696.0,
+      "reward": 0.72821044921875,
+      "reward_std": 0.00786328874528408,
+      "rewards//mean": 0.72821044921875,
+      "rewards//std": 0.03962152823805809,
+      "step": 396
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0794,
+      "grad_norm": 0.9304360151290894,
+      "kl": 0.2744011953473091,
+      "learning_rate": 4.939964841900627e-06,
+      "loss": 0.011,
+      "num_tokens": 3432248.0,
+      "reward": 0.74176025390625,
+      "reward_std": 0.012099739164113998,
+      "rewards//mean": 0.74176025390625,
+      "rewards//std": 0.030537491664290428,
+      "step": 397
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0796,
+      "grad_norm": 0.9549676775932312,
+      "kl": 0.32337421737611294,
+      "learning_rate": 4.9396187221061324e-06,
+      "loss": 0.0129,
+      "num_tokens": 3440928.0,
+      "reward": 0.7620849609375,
+      "reward_std": 0.008526656776666641,
+      "rewards//mean": 0.7620849609375,
+      "rewards//std": 0.02168460376560688,
+      "step": 398
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0798,
+      "grad_norm": 0.8273400664329529,
+      "kl": 0.30577356554567814,
+      "learning_rate": 4.939271619633508e-06,
+      "loss": 0.0122,
+      "num_tokens": 3449560.0,
+      "reward": 0.74383544921875,
+      "reward_std": 0.014349598437547684,
+      "rewards//mean": 0.74383544921875,
+      "rewards//std": 0.032983046025037766,
+      "step": 399
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.08,
+      "grad_norm": 0.9757963418960571,
+      "kl": 0.322640098631382,
+      "learning_rate": 4.938923534622567e-06,
+      "loss": 0.0129,
+      "num_tokens": 3458264.0,
+      "reward": 0.761962890625,
+      "reward_std": 0.011695911176502705,
+      "rewards//mean": 0.761962890625,
+      "rewards//std": 0.02991858497262001,
+      "step": 400
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0802,
+      "grad_norm": 0.8469364643096924,
+      "kl": 0.2975729079917073,
+      "learning_rate": 4.938574467213519e-06,
+      "loss": 0.0119,
+      "num_tokens": 3466896.0,
+      "reward": 0.76605224609375,
+      "reward_std": 0.007424627430737019,
+      "rewards//mean": 0.76605224609375,
+      "rewards//std": 0.03303670138120651,
+      "step": 401
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0804,
+      "grad_norm": 0.8685148358345032,
+      "kl": 0.3087438400834799,
+      "learning_rate": 4.938224417546965e-06,
+      "loss": 0.0123,
+      "num_tokens": 3475584.0,
+      "reward": 0.76739501953125,
+      "reward_std": 0.011819308623671532,
+      "rewards//mean": 0.76739501953125,
+      "rewards//std": 0.03834869712591171,
+      "step": 402
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0806,
+      "grad_norm": 1.04423987865448,
+      "kl": 0.2927042469382286,
+      "learning_rate": 4.937873385763909e-06,
+      "loss": 0.0117,
+      "num_tokens": 3484184.0,
+      "reward": 0.77471923828125,
+      "reward_std": 0.018968980759382248,
+      "rewards//mean": 0.77471923828125,
+      "rewards//std": 0.0364985354244709,
+      "step": 403
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0808,
+      "grad_norm": 0.994342565536499,
+      "kl": 0.250274121761322,
+      "learning_rate": 4.9375213720057435e-06,
+      "loss": 0.01,
+      "num_tokens": 3492760.0,
+      "reward": 0.76336669921875,
+      "reward_std": 0.010855021886527538,
+      "rewards//mean": 0.76336669921875,
+      "rewards//std": 0.022775080054998398,
+      "step": 404
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.081,
+      "grad_norm": 0.8562494516372681,
+      "kl": 0.241558950394392,
+      "learning_rate": 4.937168376414261e-06,
+      "loss": 0.0097,
+      "num_tokens": 3501344.0,
+      "reward": 0.7479248046875,
+      "reward_std": 0.015270461328327656,
+      "rewards//mean": 0.7479248046875,
+      "rewards//std": 0.037885088473558426,
+      "step": 405
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0812,
+      "grad_norm": 0.8906681537628174,
+      "kl": 0.2702600210905075,
+      "learning_rate": 4.9368143991316485e-06,
+      "loss": 0.0108,
+      "num_tokens": 3510008.0,
+      "reward": 0.75970458984375,
+      "reward_std": 0.011862866580486298,
+      "rewards//mean": 0.75970458984375,
+      "rewards//std": 0.03386126458644867,
+      "step": 406
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0814,
+      "grad_norm": 0.83770751953125,
+      "kl": 0.2678908761590719,
+      "learning_rate": 4.936459440300487e-06,
+      "loss": 0.0107,
+      "num_tokens": 3518752.0,
+      "reward": 0.7633056640625,
+      "reward_std": 0.010113585740327835,
+      "rewards//mean": 0.7633056640625,
+      "rewards//std": 0.026380963623523712,
+      "step": 407
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0816,
+      "grad_norm": 0.7807222604751587,
+      "kl": 0.28538877703249454,
+      "learning_rate": 4.936103500063755e-06,
+      "loss": 0.0114,
+      "num_tokens": 3527400.0,
+      "reward": 0.7591552734375,
+      "reward_std": 0.0059563172981143,
+      "rewards//mean": 0.7591552734375,
+      "rewards//std": 0.03868379816412926,
+      "step": 408
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0818,
+      "grad_norm": 0.9765918850898743,
+      "kl": 0.26162285543978214,
+      "learning_rate": 4.935746578564825e-06,
+      "loss": 0.0105,
+      "num_tokens": 3536064.0,
+      "reward": 0.7486572265625,
+      "reward_std": 0.011643504723906517,
+      "rewards//mean": 0.7486572265625,
+      "rewards//std": 0.025781990960240364,
+      "step": 409
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.082,
+      "grad_norm": 0.9713819622993469,
+      "kl": 0.2695306558161974,
+      "learning_rate": 4.935388675947463e-06,
+      "loss": 0.0108,
+      "num_tokens": 3544736.0,
+      "reward": 0.71881103515625,
+      "reward_std": 0.010069970041513443,
+      "rewards//mean": 0.71881103515625,
+      "rewards//std": 0.03545119985938072,
+      "step": 410
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0822,
+      "grad_norm": 0.9451349377632141,
+      "kl": 0.24540182575583458,
+      "learning_rate": 4.935029792355834e-06,
+      "loss": 0.0098,
+      "num_tokens": 3553424.0,
+      "reward": 0.73486328125,
+      "reward_std": 0.014077549800276756,
+      "rewards//mean": 0.73486328125,
+      "rewards//std": 0.03547174483537674,
+      "step": 411
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0824,
+      "grad_norm": 0.7770392298698425,
+      "kl": 0.2268413919955492,
+      "learning_rate": 4.934669927934496e-06,
+      "loss": 0.0091,
+      "num_tokens": 3562040.0,
+      "reward": 0.73150634765625,
+      "reward_std": 0.013487438671290874,
+      "rewards//mean": 0.73150634765625,
+      "rewards//std": 0.04288830980658531,
+      "step": 412
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0826,
+      "grad_norm": 0.674587607383728,
+      "kl": 0.28416365571320057,
+      "learning_rate": 4.9343090828284025e-06,
+      "loss": 0.0114,
+      "num_tokens": 3570696.0,
+      "reward": 0.74090576171875,
+      "reward_std": 0.010635284706950188,
+      "rewards//mean": 0.74090576171875,
+      "rewards//std": 0.03285244479775429,
+      "step": 413
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0828,
+      "grad_norm": 0.8351263403892517,
+      "kl": 0.2428603582084179,
+      "learning_rate": 4.933947257182901e-06,
+      "loss": 0.0097,
+      "num_tokens": 3579256.0,
+      "reward": 0.72833251953125,
+      "reward_std": 0.00818649772554636,
+      "rewards//mean": 0.72833251953125,
+      "rewards//std": 0.034561701118946075,
+      "step": 414
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.083,
+      "grad_norm": 0.9314544200897217,
+      "kl": 0.26274044439196587,
+      "learning_rate": 4.933584451143736e-06,
+      "loss": 0.0105,
+      "num_tokens": 3587928.0,
+      "reward": 0.72344970703125,
+      "reward_std": 0.009486508555710316,
+      "rewards//mean": 0.72344970703125,
+      "rewards//std": 0.04276353493332863,
+      "step": 415
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0832,
+      "grad_norm": 0.8246920704841614,
+      "kl": 0.2645695861428976,
+      "learning_rate": 4.933220664857045e-06,
+      "loss": 0.0106,
+      "num_tokens": 3596568.0,
+      "reward": 0.764892578125,
+      "reward_std": 0.011997243389487267,
+      "rewards//mean": 0.764892578125,
+      "rewards//std": 0.02408854104578495,
+      "step": 416
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0834,
+      "grad_norm": 0.7828191518783569,
+      "kl": 0.23742634430527687,
+      "learning_rate": 4.93285589846936e-06,
+      "loss": 0.0095,
+      "num_tokens": 3605248.0,
+      "reward": 0.77264404296875,
+      "reward_std": 0.010765868239104748,
+      "rewards//mean": 0.77264404296875,
+      "rewards//std": 0.02570635825395584,
+      "step": 417
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0836,
+      "grad_norm": 0.8400963544845581,
+      "kl": 0.2275092527270317,
+      "learning_rate": 4.932490152127611e-06,
+      "loss": 0.0091,
+      "num_tokens": 3613840.0,
+      "reward": 0.751708984375,
+      "reward_std": 0.01651906780898571,
+      "rewards//mean": 0.751708984375,
+      "rewards//std": 0.042007505893707275,
+      "step": 418
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0838,
+      "grad_norm": 0.9282857179641724,
+      "kl": 0.24514971487224102,
+      "learning_rate": 4.93212342597912e-06,
+      "loss": 0.0098,
+      "num_tokens": 3622464.0,
+      "reward": 0.7489013671875,
+      "reward_std": 0.015580926090478897,
+      "rewards//mean": 0.7489013671875,
+      "rewards//std": 0.04658199101686478,
+      "step": 419
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.084,
+      "grad_norm": 0.9025923609733582,
+      "kl": 0.23584598675370216,
+      "learning_rate": 4.931755720171603e-06,
+      "loss": 0.0094,
+      "num_tokens": 3631032.0,
+      "reward": 0.7366943359375,
+      "reward_std": 0.011178325861692429,
+      "rewards//mean": 0.7366943359375,
+      "rewards//std": 0.03618305176496506,
+      "step": 420
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0842,
+      "grad_norm": 0.7268020510673523,
+      "kl": 0.254648195579648,
+      "learning_rate": 4.931387034853173e-06,
+      "loss": 0.0102,
+      "num_tokens": 3639672.0,
+      "reward": 0.74639892578125,
+      "reward_std": 0.009266193956136703,
+      "rewards//mean": 0.74639892578125,
+      "rewards//std": 0.03361223638057709,
+      "step": 421
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0844,
+      "grad_norm": 0.8384131789207458,
+      "kl": 0.2413597498089075,
+      "learning_rate": 4.9310173701723365e-06,
+      "loss": 0.0097,
+      "num_tokens": 3648336.0,
+      "reward": 0.75970458984375,
+      "reward_std": 0.008040700107812881,
+      "rewards//mean": 0.75970458984375,
+      "rewards//std": 0.026702843606472015,
+      "step": 422
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0846,
+      "grad_norm": 0.7907002568244934,
+      "kl": 0.24139040149748325,
+      "learning_rate": 4.930646726277994e-06,
+      "loss": 0.0097,
+      "num_tokens": 3656896.0,
+      "reward": 0.7911376953125,
+      "reward_std": 0.009463133290410042,
+      "rewards//mean": 0.7911376953125,
+      "rewards//std": 0.0322941355407238,
+      "step": 423
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0848,
+      "grad_norm": 0.750485360622406,
+      "kl": 0.23888413794338703,
+      "learning_rate": 4.930275103319441e-06,
+      "loss": 0.0096,
+      "num_tokens": 3665504.0,
+      "reward": 0.73126220703125,
+      "reward_std": 0.013249299488961697,
+      "rewards//mean": 0.73126220703125,
+      "rewards//std": 0.024657992646098137,
+      "step": 424
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.085,
+      "grad_norm": 0.9737104773521423,
+      "kl": 0.2315953504294157,
+      "learning_rate": 4.9299025014463665e-06,
+      "loss": 0.0093,
+      "num_tokens": 3674080.0,
+      "reward": 0.7303466796875,
+      "reward_std": 0.013063129037618637,
+      "rewards//mean": 0.7303466796875,
+      "rewards//std": 0.0421634316444397,
+      "step": 425
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0852,
+      "grad_norm": 0.7097475528717041,
+      "kl": 0.2388191670179367,
+      "learning_rate": 4.9295289208088545e-06,
+      "loss": 0.0096,
+      "num_tokens": 3682640.0,
+      "reward": 0.74786376953125,
+      "reward_std": 0.009970373474061489,
+      "rewards//mean": 0.74786376953125,
+      "rewards//std": 0.03215102478861809,
+      "step": 426
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0854,
+      "grad_norm": 0.78131502866745,
+      "kl": 0.2351592220366001,
+      "learning_rate": 4.929154361557384e-06,
+      "loss": 0.0094,
+      "num_tokens": 3691304.0,
+      "reward": 0.77752685546875,
+      "reward_std": 0.010701300576329231,
+      "rewards//mean": 0.77752685546875,
+      "rewards//std": 0.025538571178913116,
+      "step": 427
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0856,
+      "grad_norm": 0.813309907913208,
+      "kl": 0.2603413835167885,
+      "learning_rate": 4.928778823842828e-06,
+      "loss": 0.0104,
+      "num_tokens": 3700024.0,
+      "reward": 0.75299072265625,
+      "reward_std": 0.007520100101828575,
+      "rewards//mean": 0.75299072265625,
+      "rewards//std": 0.03588072210550308,
+      "step": 428
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0858,
+      "grad_norm": 0.7221551537513733,
+      "kl": 0.18088798597455025,
+      "learning_rate": 4.928402307816452e-06,
+      "loss": 0.0072,
+      "num_tokens": 3708784.0,
+      "reward": 0.74078369140625,
+      "reward_std": 0.01760246977210045,
+      "rewards//mean": 0.74078369140625,
+      "rewards//std": 0.04441485553979874,
+      "step": 429
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.086,
+      "grad_norm": 0.8385155200958252,
+      "kl": 0.21827073767781258,
+      "learning_rate": 4.928024813629917e-06,
+      "loss": 0.0087,
+      "num_tokens": 3717392.0,
+      "reward": 0.7694091796875,
+      "reward_std": 0.009142270311713219,
+      "rewards//mean": 0.7694091796875,
+      "rewards//std": 0.026874415576457977,
+      "step": 430
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0862,
+      "grad_norm": 0.6926012635231018,
+      "kl": 0.22130563855171204,
+      "learning_rate": 4.927646341435276e-06,
+      "loss": 0.0089,
+      "num_tokens": 3726008.0,
+      "reward": 0.75091552734375,
+      "reward_std": 0.010623462498188019,
+      "rewards//mean": 0.75091552734375,
+      "rewards//std": 0.028200527653098106,
+      "step": 431
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0864,
+      "grad_norm": 0.8035763502120972,
+      "kl": 0.252463411539793,
+      "learning_rate": 4.92726689138498e-06,
+      "loss": 0.0101,
+      "num_tokens": 3734584.0,
+      "reward": 0.74298095703125,
+      "reward_std": 0.011688274331390858,
+      "rewards//mean": 0.74298095703125,
+      "rewards//std": 0.026449372991919518,
+      "step": 432
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0866,
+      "grad_norm": 0.6842226982116699,
+      "kl": 0.23155632801353931,
+      "learning_rate": 4.92688646363187e-06,
+      "loss": 0.0093,
+      "num_tokens": 3743296.0,
+      "reward": 0.7818603515625,
+      "reward_std": 0.011189782060682774,
+      "rewards//mean": 0.7818603515625,
+      "rewards//std": 0.033827103674411774,
+      "step": 433
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0868,
+      "grad_norm": 0.6330403089523315,
+      "kl": 0.21206197701394558,
+      "learning_rate": 4.926505058329184e-06,
+      "loss": 0.0085,
+      "num_tokens": 3752000.0,
+      "reward": 0.7442626953125,
+      "reward_std": 0.01008138619363308,
+      "rewards//mean": 0.7442626953125,
+      "rewards//std": 0.02858632244169712,
+      "step": 434
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.087,
+      "grad_norm": 0.877251386642456,
+      "kl": 0.22307201102375984,
+      "learning_rate": 4.9261226756305495e-06,
+      "loss": 0.0089,
+      "num_tokens": 3760584.0,
+      "reward": 0.71490478515625,
+      "reward_std": 0.006664145737886429,
+      "rewards//mean": 0.71490478515625,
+      "rewards//std": 0.04637888818979263,
+      "step": 435
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0872,
+      "grad_norm": 0.6964420676231384,
+      "kl": 0.2119651883840561,
+      "learning_rate": 4.925739315689991e-06,
+      "loss": 0.0085,
+      "num_tokens": 3769168.0,
+      "reward": 0.7415771484375,
+      "reward_std": 0.00940138753503561,
+      "rewards//mean": 0.7415771484375,
+      "rewards//std": 0.03519562631845474,
+      "step": 436
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0874,
+      "grad_norm": 0.737678587436676,
+      "kl": 0.22590421885252,
+      "learning_rate": 4.925354978661928e-06,
+      "loss": 0.009,
+      "num_tokens": 3777808.0,
+      "reward": 0.7454833984375,
+      "reward_std": 0.008858283050358295,
+      "rewards//mean": 0.7454833984375,
+      "rewards//std": 0.027649568393826485,
+      "step": 437
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0876,
+      "grad_norm": 0.6990619897842407,
+      "kl": 0.21303631737828255,
+      "learning_rate": 4.924969664701168e-06,
+      "loss": 0.0085,
+      "num_tokens": 3786392.0,
+      "reward": 0.74658203125,
+      "reward_std": 0.011121334508061409,
+      "rewards//mean": 0.74658203125,
+      "rewards//std": 0.025176284834742546,
+      "step": 438
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0878,
+      "grad_norm": 0.6292837858200073,
+      "kl": 0.21526087448000908,
+      "learning_rate": 4.924583373962918e-06,
+      "loss": 0.0086,
+      "num_tokens": 3795064.0,
+      "reward": 0.7734375,
+      "reward_std": 0.010552143678069115,
+      "rewards//mean": 0.7734375,
+      "rewards//std": 0.02722390927374363,
+      "step": 439
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.088,
+      "grad_norm": 0.6078249216079712,
+      "kl": 0.24454114213585854,
+      "learning_rate": 4.924196106602774e-06,
+      "loss": 0.0098,
+      "num_tokens": 3803632.0,
+      "reward": 0.74200439453125,
+      "reward_std": 0.009583698585629463,
+      "rewards//mean": 0.74200439453125,
+      "rewards//std": 0.053606946021318436,
+      "step": 440
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0882,
+      "grad_norm": 0.7330633997917175,
+      "kl": 0.2058623656630516,
+      "learning_rate": 4.9238078627767285e-06,
+      "loss": 0.0082,
+      "num_tokens": 3812296.0,
+      "reward": 0.77801513671875,
+      "reward_std": 0.01102867629379034,
+      "rewards//mean": 0.77801513671875,
+      "rewards//std": 0.026662563905119896,
+      "step": 441
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0884,
+      "grad_norm": 0.7553256154060364,
+      "kl": 0.21089842356741428,
+      "learning_rate": 4.923418642641166e-06,
+      "loss": 0.0084,
+      "num_tokens": 3820952.0,
+      "reward": 0.7528076171875,
+      "reward_std": 0.011244535446166992,
+      "rewards//mean": 0.7528076171875,
+      "rewards//std": 0.039164479821920395,
+      "step": 442
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0886,
+      "grad_norm": 0.7146939039230347,
+      "kl": 0.22701359912753105,
+      "learning_rate": 4.923028446352864e-06,
+      "loss": 0.0091,
+      "num_tokens": 3829712.0,
+      "reward": 0.71051025390625,
+      "reward_std": 0.016265802085399628,
+      "rewards//mean": 0.71051025390625,
+      "rewards//std": 0.048049308359622955,
+      "step": 443
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0888,
+      "grad_norm": 0.807270884513855,
+      "kl": 0.25617854483425617,
+      "learning_rate": 4.922637274068993e-06,
+      "loss": 0.0102,
+      "num_tokens": 3838296.0,
+      "reward": 0.72760009765625,
+      "reward_std": 0.008829087018966675,
+      "rewards//mean": 0.72760009765625,
+      "rewards//std": 0.017555613070726395,
+      "step": 444
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.089,
+      "grad_norm": 0.8020032048225403,
+      "kl": 0.2054343856871128,
+      "learning_rate": 4.9222451259471185e-06,
+      "loss": 0.0082,
+      "num_tokens": 3846960.0,
+      "reward": 0.74591064453125,
+      "reward_std": 0.008869110606610775,
+      "rewards//mean": 0.74591064453125,
+      "rewards//std": 0.029458094388246536,
+      "step": 445
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0892,
+      "grad_norm": 0.6758546233177185,
+      "kl": 0.22217164561152458,
+      "learning_rate": 4.921852002145196e-06,
+      "loss": 0.0089,
+      "num_tokens": 3855688.0,
+      "reward": 0.7484130859375,
+      "reward_std": 0.008662059903144836,
+      "rewards//mean": 0.7484130859375,
+      "rewards//std": 0.0318770706653595,
+      "step": 446
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0894,
+      "grad_norm": 0.8543888926506042,
+      "kl": 0.24924145638942719,
+      "learning_rate": 4.921457902821578e-06,
+      "loss": 0.01,
+      "num_tokens": 3864312.0,
+      "reward": 0.759765625,
+      "reward_std": 0.017270008102059364,
+      "rewards//mean": 0.759765625,
+      "rewards//std": 0.029421651735901833,
+      "step": 447
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0896,
+      "grad_norm": 0.6530463695526123,
+      "kl": 0.2602735925465822,
+      "learning_rate": 4.921062828135006e-06,
+      "loss": 0.0104,
+      "num_tokens": 3872992.0,
+      "reward": 0.75787353515625,
+      "reward_std": 0.015410242602229118,
+      "rewards//mean": 0.75787353515625,
+      "rewards//std": 0.02605554088950157,
+      "step": 448
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0898,
+      "grad_norm": 0.9578370451927185,
+      "kl": 0.2606674674898386,
+      "learning_rate": 4.920666778244616e-06,
+      "loss": 0.0104,
+      "num_tokens": 3881712.0,
+      "reward": 0.77020263671875,
+      "reward_std": 0.008511267602443695,
+      "rewards//mean": 0.77020263671875,
+      "rewards//std": 0.030421772971749306,
+      "step": 449
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.09,
+      "grad_norm": 0.6777032017707825,
+      "kl": 0.2329980656504631,
+      "learning_rate": 4.920269753309937e-06,
+      "loss": 0.0093,
+      "num_tokens": 3890312.0,
+      "reward": 0.7421875,
+      "reward_std": 0.00921049527823925,
+      "rewards//mean": 0.7421875,
+      "rewards//std": 0.031932346522808075,
+      "step": 450
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0902,
+      "grad_norm": 0.7345518469810486,
+      "kl": 0.23562784306704998,
+      "learning_rate": 4.919871753490892e-06,
+      "loss": 0.0094,
+      "num_tokens": 3899016.0,
+      "reward": 0.73236083984375,
+      "reward_std": 0.009874099865555763,
+      "rewards//mean": 0.73236083984375,
+      "rewards//std": 0.04835017770528793,
+      "step": 451
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0904,
+      "grad_norm": 0.6494887471199036,
+      "kl": 0.22641596291214228,
+      "learning_rate": 4.919472778947793e-06,
+      "loss": 0.0091,
+      "num_tokens": 3907752.0,
+      "reward": 0.7728271484375,
+      "reward_std": 0.008734005503356457,
+      "rewards//mean": 0.7728271484375,
+      "rewards//std": 0.019265538081526756,
+      "step": 452
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0906,
+      "grad_norm": 0.6345562934875488,
+      "kl": 0.22991832438856363,
+      "learning_rate": 4.919072829841347e-06,
+      "loss": 0.0092,
+      "num_tokens": 3916496.0,
+      "reward": 0.76580810546875,
+      "reward_std": 0.009036125615239143,
+      "rewards//mean": 0.76580810546875,
+      "rewards//std": 0.03826966881752014,
+      "step": 453
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0908,
+      "grad_norm": 0.7087083458900452,
+      "kl": 0.23438394255936146,
+      "learning_rate": 4.918671906332656e-06,
+      "loss": 0.0094,
+      "num_tokens": 3925008.0,
+      "reward": 0.732666015625,
+      "reward_std": 0.011662309989333153,
+      "rewards//mean": 0.732666015625,
+      "rewards//std": 0.034244511276483536,
+      "step": 454
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.091,
+      "grad_norm": 0.7872934937477112,
+      "kl": 0.2746329791843891,
+      "learning_rate": 4.91827000858321e-06,
+      "loss": 0.011,
+      "num_tokens": 3933696.0,
+      "reward": 0.73626708984375,
+      "reward_std": 0.007544973865151405,
+      "rewards//mean": 0.73626708984375,
+      "rewards//std": 0.03399689868092537,
+      "step": 455
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0912,
+      "grad_norm": 0.6922155022621155,
+      "kl": 0.22344142571091652,
+      "learning_rate": 4.917867136754894e-06,
+      "loss": 0.0089,
+      "num_tokens": 3942304.0,
+      "reward": 0.77667236328125,
+      "reward_std": 0.013890949077904224,
+      "rewards//mean": 0.77667236328125,
+      "rewards//std": 0.032455623149871826,
+      "step": 456
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0914,
+      "grad_norm": 0.8709951043128967,
+      "kl": 0.29020215198397636,
+      "learning_rate": 4.917463291009984e-06,
+      "loss": 0.0116,
+      "num_tokens": 3950968.0,
+      "reward": 0.750244140625,
+      "reward_std": 0.00925417710095644,
+      "rewards//mean": 0.750244140625,
+      "rewards//std": 0.02082025073468685,
+      "step": 457
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0916,
+      "grad_norm": 0.7772853374481201,
+      "kl": 0.2596856001764536,
+      "learning_rate": 4.917058471511149e-06,
+      "loss": 0.0104,
+      "num_tokens": 3959640.0,
+      "reward": 0.74822998046875,
+      "reward_std": 0.012660522013902664,
+      "rewards//mean": 0.74822998046875,
+      "rewards//std": 0.03531729802489281,
+      "step": 458
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0918,
+      "grad_norm": 0.9631777405738831,
+      "kl": 0.2647391799837351,
+      "learning_rate": 4.916652678421451e-06,
+      "loss": 0.0106,
+      "num_tokens": 3968256.0,
+      "reward": 0.76495361328125,
+      "reward_std": 0.008287420496344566,
+      "rewards//mean": 0.76495361328125,
+      "rewards//std": 0.021716255694627762,
+      "step": 459
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.092,
+      "grad_norm": 0.6951785683631897,
+      "kl": 0.24628694355487823,
+      "learning_rate": 4.916245911904344e-06,
+      "loss": 0.0099,
+      "num_tokens": 3977024.0,
+      "reward": 0.772216796875,
+      "reward_std": 0.012418104335665703,
+      "rewards//mean": 0.772216796875,
+      "rewards//std": 0.024899380281567574,
+      "step": 460
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0922,
+      "grad_norm": 0.7456372976303101,
+      "kl": 0.25542332231998444,
+      "learning_rate": 4.9158381721236715e-06,
+      "loss": 0.0102,
+      "num_tokens": 3985664.0,
+      "reward": 0.72442626953125,
+      "reward_std": 0.011360350996255875,
+      "rewards//mean": 0.72442626953125,
+      "rewards//std": 0.03637805953621864,
+      "step": 461
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0924,
+      "grad_norm": 0.7007219791412354,
+      "kl": 0.2571563795208931,
+      "learning_rate": 4.915429459243673e-06,
+      "loss": 0.0103,
+      "num_tokens": 3994480.0,
+      "reward": 0.75836181640625,
+      "reward_std": 0.00619722343981266,
+      "rewards//mean": 0.75836181640625,
+      "rewards//std": 0.025561677291989326,
+      "step": 462
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0926,
+      "grad_norm": 0.6913952231407166,
+      "kl": 0.2788440138101578,
+      "learning_rate": 4.9150197734289764e-06,
+      "loss": 0.0112,
+      "num_tokens": 4003112.0,
+      "reward": 0.7447509765625,
+      "reward_std": 0.007206355221569538,
+      "rewards//mean": 0.7447509765625,
+      "rewards//std": 0.03733934462070465,
+      "step": 463
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0928,
+      "grad_norm": 0.675417423248291,
+      "kl": 0.23573023825883865,
+      "learning_rate": 4.9146091148446055e-06,
+      "loss": 0.0094,
+      "num_tokens": 4011720.0,
+      "reward": 0.7464599609375,
+      "reward_std": 0.015771940350532532,
+      "rewards//mean": 0.7464599609375,
+      "rewards//std": 0.04249957576394081,
+      "step": 464
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.093,
+      "grad_norm": 0.7302259802818298,
+      "kl": 0.27733396738767624,
+      "learning_rate": 4.91419748365597e-06,
+      "loss": 0.0111,
+      "num_tokens": 4020384.0,
+      "reward": 0.734130859375,
+      "reward_std": 0.007520634680986404,
+      "rewards//mean": 0.734130859375,
+      "rewards//std": 0.03848075866699219,
+      "step": 465
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0932,
+      "grad_norm": 0.7138959169387817,
+      "kl": 0.2417994262650609,
+      "learning_rate": 4.9137848800288775e-06,
+      "loss": 0.0097,
+      "num_tokens": 4029016.0,
+      "reward": 0.75616455078125,
+      "reward_std": 0.010032439604401588,
+      "rewards//mean": 0.75616455078125,
+      "rewards//std": 0.02122269943356514,
+      "step": 466
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0934,
+      "grad_norm": 0.7322536706924438,
+      "kl": 0.27695489302277565,
+      "learning_rate": 4.9133713041295235e-06,
+      "loss": 0.0111,
+      "num_tokens": 4037688.0,
+      "reward": 0.7783203125,
+      "reward_std": 0.009051885455846786,
+      "rewards//mean": 0.7783203125,
+      "rewards//std": 0.03194751217961311,
+      "step": 467
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0936,
+      "grad_norm": 0.8252130746841431,
+      "kl": 0.26474233716726303,
+      "learning_rate": 4.912956756124498e-06,
+      "loss": 0.0106,
+      "num_tokens": 4046304.0,
+      "reward": 0.7481689453125,
+      "reward_std": 0.0064436085522174835,
+      "rewards//mean": 0.7481689453125,
+      "rewards//std": 0.0320381224155426,
+      "step": 468
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0938,
+      "grad_norm": 0.6932944059371948,
+      "kl": 0.27318440936505795,
+      "learning_rate": 4.912541236180779e-06,
+      "loss": 0.0109,
+      "num_tokens": 4054992.0,
+      "reward": 0.71624755859375,
+      "reward_std": 0.009076687507331371,
+      "rewards//mean": 0.71624755859375,
+      "rewards//std": 0.03491336852312088,
+      "step": 469
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.094,
+      "grad_norm": 0.6723066568374634,
+      "kl": 0.2577223964035511,
+      "learning_rate": 4.9121247444657384e-06,
+      "loss": 0.0103,
+      "num_tokens": 4063624.0,
+      "reward": 0.7515869140625,
+      "reward_std": 0.005873112007975578,
+      "rewards//mean": 0.7515869140625,
+      "rewards//std": 0.02327904850244522,
+      "step": 470
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0942,
+      "grad_norm": 0.8410773277282715,
+      "kl": 0.2808041274547577,
+      "learning_rate": 4.91170728114714e-06,
+      "loss": 0.0112,
+      "num_tokens": 4072256.0,
+      "reward": 0.76995849609375,
+      "reward_std": 0.005484414286911488,
+      "rewards//mean": 0.76995849609375,
+      "rewards//std": 0.022884486243128777,
+      "step": 471
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0944,
+      "grad_norm": 0.6858919262886047,
+      "kl": 0.2676295880228281,
+      "learning_rate": 4.911288846393136e-06,
+      "loss": 0.0107,
+      "num_tokens": 4080920.0,
+      "reward": 0.742431640625,
+      "reward_std": 0.012072248384356499,
+      "rewards//mean": 0.742431640625,
+      "rewards//std": 0.022215967997908592,
+      "step": 472
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0946,
+      "grad_norm": 0.6118029356002808,
+      "kl": 0.21254741679877043,
+      "learning_rate": 4.910869440372274e-06,
+      "loss": 0.0085,
+      "num_tokens": 4089496.0,
+      "reward": 0.739990234375,
+      "reward_std": 0.01614592783153057,
+      "rewards//mean": 0.739990234375,
+      "rewards//std": 0.03497926890850067,
+      "step": 473
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0948,
+      "grad_norm": 0.7061100602149963,
+      "kl": 0.25430491380393505,
+      "learning_rate": 4.910449063253489e-06,
+      "loss": 0.0102,
+      "num_tokens": 4098072.0,
+      "reward": 0.75494384765625,
+      "reward_std": 0.01261814869940281,
+      "rewards//mean": 0.75494384765625,
+      "rewards//std": 0.03197965770959854,
+      "step": 474
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.095,
+      "grad_norm": 0.6372392773628235,
+      "kl": 0.29845254495739937,
+      "learning_rate": 4.9100277152061105e-06,
+      "loss": 0.0119,
+      "num_tokens": 4106672.0,
+      "reward": 0.7095947265625,
+      "reward_std": 0.009667182341217995,
+      "rewards//mean": 0.7095947265625,
+      "rewards//std": 0.045762356370687485,
+      "step": 475
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0952,
+      "grad_norm": 0.6701680421829224,
+      "kl": 0.25746934674680233,
+      "learning_rate": 4.9096053963998555e-06,
+      "loss": 0.0103,
+      "num_tokens": 4115384.0,
+      "reward": 0.742919921875,
+      "reward_std": 0.01074405387043953,
+      "rewards//mean": 0.742919921875,
+      "rewards//std": 0.027916818857192993,
+      "step": 476
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0954,
+      "grad_norm": 0.8074530363082886,
+      "kl": 0.2788533419370651,
+      "learning_rate": 4.909182107004835e-06,
+      "loss": 0.0112,
+      "num_tokens": 4124064.0,
+      "reward": 0.73681640625,
+      "reward_std": 0.006065527442842722,
+      "rewards//mean": 0.73681640625,
+      "rewards//std": 0.03234307840466499,
+      "step": 477
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0956,
+      "grad_norm": 0.7189854979515076,
+      "kl": 0.27882962487637997,
+      "learning_rate": 4.908757847191551e-06,
+      "loss": 0.0112,
+      "num_tokens": 4132704.0,
+      "reward": 0.74261474609375,
+      "reward_std": 0.01600596494972706,
+      "rewards//mean": 0.74261474609375,
+      "rewards//std": 0.043848197907209396,
+      "step": 478
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0958,
+      "grad_norm": 0.7320364117622375,
+      "kl": 0.2482055276632309,
+      "learning_rate": 4.908332617130893e-06,
+      "loss": 0.0099,
+      "num_tokens": 4141304.0,
+      "reward": 0.76947021484375,
+      "reward_std": 0.007574299816042185,
+      "rewards//mean": 0.76947021484375,
+      "rewards//std": 0.030989211052656174,
+      "step": 479
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.096,
+      "grad_norm": 0.735087513923645,
+      "kl": 0.24100211029872298,
+      "learning_rate": 4.907906416994146e-06,
+      "loss": 0.0096,
+      "num_tokens": 4149840.0,
+      "reward": 0.77349853515625,
+      "reward_std": 0.013887856155633926,
+      "rewards//mean": 0.77349853515625,
+      "rewards//std": 0.03502505645155907,
+      "step": 480
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0962,
+      "grad_norm": 0.693162739276886,
+      "kl": 0.27953667007386684,
+      "learning_rate": 4.907479246952981e-06,
+      "loss": 0.0112,
+      "num_tokens": 4158448.0,
+      "reward": 0.718017578125,
+      "reward_std": 0.00825615506619215,
+      "rewards//mean": 0.718017578125,
+      "rewards//std": 0.03687366470694542,
+      "step": 481
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0964,
+      "grad_norm": 0.6603836417198181,
+      "kl": 0.23947274032980204,
+      "learning_rate": 4.907051107179464e-06,
+      "loss": 0.0096,
+      "num_tokens": 4167072.0,
+      "reward": 0.748046875,
+      "reward_std": 0.013273896649479866,
+      "rewards//mean": 0.748046875,
+      "rewards//std": 0.0389009565114975,
+      "step": 482
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0966,
+      "grad_norm": 0.6945295929908752,
+      "kl": 0.2583098318427801,
+      "learning_rate": 4.9066219978460485e-06,
+      "loss": 0.0103,
+      "num_tokens": 4175656.0,
+      "reward": 0.763916015625,
+      "reward_std": 0.007394441869109869,
+      "rewards//mean": 0.763916015625,
+      "rewards//std": 0.03223337233066559,
+      "step": 483
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0968,
+      "grad_norm": 0.6192371845245361,
+      "kl": 0.2828360088169575,
+      "learning_rate": 4.90619191912558e-06,
+      "loss": 0.0113,
+      "num_tokens": 4184232.0,
+      "reward": 0.7418212890625,
+      "reward_std": 0.00855693407356739,
+      "rewards//mean": 0.7418212890625,
+      "rewards//std": 0.02327904850244522,
+      "step": 484
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.097,
+      "grad_norm": 0.6635370850563049,
+      "kl": 0.2756698988378048,
+      "learning_rate": 4.905760871191295e-06,
+      "loss": 0.011,
+      "num_tokens": 4192904.0,
+      "reward": 0.71905517578125,
+      "reward_std": 0.007833951152861118,
+      "rewards//mean": 0.71905517578125,
+      "rewards//std": 0.04106537625193596,
+      "step": 485
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0972,
+      "grad_norm": 0.669340968132019,
+      "kl": 0.2513146288692951,
+      "learning_rate": 4.9053288542168185e-06,
+      "loss": 0.0101,
+      "num_tokens": 4201520.0,
+      "reward": 0.7288818359375,
+      "reward_std": 0.011037398129701614,
+      "rewards//mean": 0.7288818359375,
+      "rewards//std": 0.022028131410479546,
+      "step": 486
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0974,
+      "grad_norm": 0.6306434273719788,
+      "kl": 0.2504331525415182,
+      "learning_rate": 4.904895868376167e-06,
+      "loss": 0.01,
+      "num_tokens": 4210128.0,
+      "reward": 0.76690673828125,
+      "reward_std": 0.010853402316570282,
+      "rewards//mean": 0.76690673828125,
+      "rewards//std": 0.03223472461104393,
+      "step": 487
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0976,
+      "grad_norm": 0.6316109299659729,
+      "kl": 0.24033397436141968,
+      "learning_rate": 4.904461913843747e-06,
+      "loss": 0.0096,
+      "num_tokens": 4218696.0,
+      "reward": 0.73101806640625,
+      "reward_std": 0.009801981970667839,
+      "rewards//mean": 0.73101806640625,
+      "rewards//std": 0.033093471080064774,
+      "step": 488
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0978,
+      "grad_norm": 0.692654013633728,
+      "kl": 0.30100347846746445,
+      "learning_rate": 4.904026990794356e-06,
+      "loss": 0.012,
+      "num_tokens": 4227304.0,
+      "reward": 0.7596435546875,
+      "reward_std": 0.009308423846960068,
+      "rewards//mean": 0.7596435546875,
+      "rewards//std": 0.032158851623535156,
+      "step": 489
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.098,
+      "grad_norm": 0.6925479769706726,
+      "kl": 0.26741546019911766,
+      "learning_rate": 4.903591099403181e-06,
+      "loss": 0.0107,
+      "num_tokens": 4235968.0,
+      "reward": 0.7447509765625,
+      "reward_std": 0.01731371134519577,
+      "rewards//mean": 0.7447509765625,
+      "rewards//std": 0.035796087235212326,
+      "step": 490
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0982,
+      "grad_norm": 0.5992981791496277,
+      "kl": 0.24271851778030396,
+      "learning_rate": 4.903154239845798e-06,
+      "loss": 0.0097,
+      "num_tokens": 4244616.0,
+      "reward": 0.73974609375,
+      "reward_std": 0.010727489367127419,
+      "rewards//mean": 0.73974609375,
+      "rewards//std": 0.02847197651863098,
+      "step": 491
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0984,
+      "grad_norm": 0.6184040307998657,
+      "kl": 0.24116972647607327,
+      "learning_rate": 4.902716412298174e-06,
+      "loss": 0.0096,
+      "num_tokens": 4253168.0,
+      "reward": 0.72052001953125,
+      "reward_std": 0.011508103460073471,
+      "rewards//mean": 0.72052001953125,
+      "rewards//std": 0.03347956761717796,
+      "step": 492
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0986,
+      "grad_norm": 0.6333391070365906,
+      "kl": 0.2607311652973294,
+      "learning_rate": 4.902277616936667e-06,
+      "loss": 0.0104,
+      "num_tokens": 4261768.0,
+      "reward": 0.7559814453125,
+      "reward_std": 0.009352114051580429,
+      "rewards//mean": 0.7559814453125,
+      "rewards//std": 0.030678750947117805,
+      "step": 493
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0988,
+      "grad_norm": 0.700835108757019,
+      "kl": 0.2442559963092208,
+      "learning_rate": 4.901837853938024e-06,
+      "loss": 0.0098,
+      "num_tokens": 4270328.0,
+      "reward": 0.7232666015625,
+      "reward_std": 0.011704965494573116,
+      "rewards//mean": 0.7232666015625,
+      "rewards//std": 0.039587363600730896,
+      "step": 494
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.099,
+      "grad_norm": 0.687675952911377,
+      "kl": 0.28429468162357807,
+      "learning_rate": 4.90139712347938e-06,
+      "loss": 0.0114,
+      "num_tokens": 4279016.0,
+      "reward": 0.74224853515625,
+      "reward_std": 0.006493415683507919,
+      "rewards//mean": 0.74224853515625,
+      "rewards//std": 0.02292017824947834,
+      "step": 495
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0992,
+      "grad_norm": 0.6724541187286377,
+      "kl": 0.2627929784357548,
+      "learning_rate": 4.900955425738262e-06,
+      "loss": 0.0105,
+      "num_tokens": 4287632.0,
+      "reward": 0.73876953125,
+      "reward_std": 0.008314857259392738,
+      "rewards//mean": 0.73876953125,
+      "rewards//std": 0.032192960381507874,
+      "step": 496
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0994,
+      "grad_norm": 0.6971147656440735,
+      "kl": 0.2989068515598774,
+      "learning_rate": 4.900512760892585e-06,
+      "loss": 0.012,
+      "num_tokens": 4296256.0,
+      "reward": 0.73370361328125,
+      "reward_std": 0.009461992420256138,
+      "rewards//mean": 0.73370361328125,
+      "rewards//std": 0.025719311088323593,
+      "step": 497
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0996,
+      "grad_norm": 0.7619755864143372,
+      "kl": 0.28933385387063026,
+      "learning_rate": 4.900069129120656e-06,
+      "loss": 0.0116,
+      "num_tokens": 4304944.0,
+      "reward": 0.7659912109375,
+      "reward_std": 0.009017490781843662,
+      "rewards//mean": 0.7659912109375,
+      "rewards//std": 0.03981613367795944,
+      "step": 498
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0998,
+      "grad_norm": 0.7067105174064636,
+      "kl": 0.2626380883157253,
+      "learning_rate": 4.899624530601168e-06,
+      "loss": 0.0105,
+      "num_tokens": 4313560.0,
+      "reward": 0.72686767578125,
+      "reward_std": 0.01192802656441927,
+      "rewards//mean": 0.72686767578125,
+      "rewards//std": 0.037255480885505676,
+      "step": 499
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1,
+      "grad_norm": 0.7758896946907043,
+      "kl": 0.28313127160072327,
+      "learning_rate": 4.899178965513206e-06,
+      "loss": 0.0113,
+      "num_tokens": 4322208.0,
+      "reward": 0.76763916015625,
+      "reward_std": 0.0051316977478563786,
+      "rewards//mean": 0.76763916015625,
+      "rewards//std": 0.022271685302257538,
+      "step": 500
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1002,
+      "grad_norm": 0.7543128132820129,
+      "kl": 0.29646800085902214,
+      "learning_rate": 4.8987324340362445e-06,
+      "loss": 0.0119,
+      "num_tokens": 4330888.0,
+      "reward": 0.74884033203125,
+      "reward_std": 0.007682453375309706,
+      "rewards//mean": 0.74884033203125,
+      "rewards//std": 0.027051476761698723,
+      "step": 501
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1004,
+      "grad_norm": 0.7896819114685059,
+      "kl": 0.33157695457339287,
+      "learning_rate": 4.898284936350144e-06,
+      "loss": 0.0133,
+      "num_tokens": 4339480.0,
+      "reward": 0.732666015625,
+      "reward_std": 0.007414411753416061,
+      "rewards//mean": 0.732666015625,
+      "rewards//std": 0.03544698655605316,
+      "step": 502
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1006,
+      "grad_norm": 0.6674100160598755,
+      "kl": 0.26767886988818645,
+      "learning_rate": 4.897836472635159e-06,
+      "loss": 0.0107,
+      "num_tokens": 4348208.0,
+      "reward": 0.71868896484375,
+      "reward_std": 0.0074168480932712555,
+      "rewards//mean": 0.71868896484375,
+      "rewards//std": 0.03925728052854538,
+      "step": 503
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1008,
+      "grad_norm": 0.7596335411071777,
+      "kl": 0.2823500316590071,
+      "learning_rate": 4.89738704307193e-06,
+      "loss": 0.0113,
+      "num_tokens": 4356904.0,
+      "reward": 0.748046875,
+      "reward_std": 0.005976288113743067,
+      "rewards//mean": 0.748046875,
+      "rewards//std": 0.02426011860370636,
+      "step": 504
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.101,
+      "grad_norm": 0.7461748123168945,
+      "kl": 0.2936761640012264,
+      "learning_rate": 4.896936647841485e-06,
+      "loss": 0.0117,
+      "num_tokens": 4365528.0,
+      "reward": 0.75103759765625,
+      "reward_std": 0.011169584468007088,
+      "rewards//mean": 0.75103759765625,
+      "rewards//std": 0.030783364549279213,
+      "step": 505
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1012,
+      "grad_norm": 0.7608941197395325,
+      "kl": 0.28775020502507687,
+      "learning_rate": 4.896485287125247e-06,
+      "loss": 0.0115,
+      "num_tokens": 4374120.0,
+      "reward": 0.73614501953125,
+      "reward_std": 0.008655678480863571,
+      "rewards//mean": 0.73614501953125,
+      "rewards//std": 0.02487071417272091,
+      "step": 506
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1014,
+      "grad_norm": 0.8823245167732239,
+      "kl": 0.2823618911206722,
+      "learning_rate": 4.896032961105021e-06,
+      "loss": 0.0113,
+      "num_tokens": 4382808.0,
+      "reward": 0.75872802734375,
+      "reward_std": 0.007658984046429396,
+      "rewards//mean": 0.75872802734375,
+      "rewards//std": 0.02521706186234951,
+      "step": 507
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1016,
+      "grad_norm": 0.6119314432144165,
+      "kl": 0.2920355089008808,
+      "learning_rate": 4.8955796699630045e-06,
+      "loss": 0.0117,
+      "num_tokens": 4391536.0,
+      "reward": 0.767578125,
+      "reward_std": 0.006091872230172157,
+      "rewards//mean": 0.767578125,
+      "rewards//std": 0.02380661852657795,
+      "step": 508
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1018,
+      "grad_norm": 0.6585849523544312,
+      "kl": 0.2887584716081619,
+      "learning_rate": 4.895125413881783e-06,
+      "loss": 0.0116,
+      "num_tokens": 4400152.0,
+      "reward": 0.748291015625,
+      "reward_std": 0.005973452236503363,
+      "rewards//mean": 0.748291015625,
+      "rewards//std": 0.030804071575403214,
+      "step": 509
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.102,
+      "grad_norm": 0.7349271178245544,
+      "kl": 0.27199224196374416,
+      "learning_rate": 4.894670193044332e-06,
+      "loss": 0.0109,
+      "num_tokens": 4408712.0,
+      "reward": 0.72711181640625,
+      "reward_std": 0.0059516276232898235,
+      "rewards//mean": 0.72711181640625,
+      "rewards//std": 0.03458928316831589,
+      "step": 510
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1022,
+      "grad_norm": 0.6272158026695251,
+      "kl": 0.29213687032461166,
+      "learning_rate": 4.894214007634014e-06,
+      "loss": 0.0117,
+      "num_tokens": 4417376.0,
+      "reward": 0.763427734375,
+      "reward_std": 0.00552313681691885,
+      "rewards//mean": 0.763427734375,
+      "rewards//std": 0.02124631404876709,
+      "step": 511
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1024,
+      "grad_norm": 0.7491189241409302,
+      "kl": 0.2788054086267948,
+      "learning_rate": 4.893756857834579e-06,
+      "loss": 0.0112,
+      "num_tokens": 4426096.0,
+      "reward": 0.75390625,
+      "reward_std": 0.015128916129469872,
+      "rewards//mean": 0.75390625,
+      "rewards//std": 0.04438012093305588,
+      "step": 512
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1026,
+      "grad_norm": 0.598261833190918,
+      "kl": 0.2626344934105873,
+      "learning_rate": 4.893298743830168e-06,
+      "loss": 0.0105,
+      "num_tokens": 4434696.0,
+      "reward": 0.74615478515625,
+      "reward_std": 0.012900955975055695,
+      "rewards//mean": 0.74615478515625,
+      "rewards//std": 0.03896622732281685,
+      "step": 513
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1028,
+      "grad_norm": 0.6159838438034058,
+      "kl": 0.2789937425404787,
+      "learning_rate": 4.89283966580531e-06,
+      "loss": 0.0112,
+      "num_tokens": 4443352.0,
+      "reward": 0.7607421875,
+      "reward_std": 0.0056966980919241905,
+      "rewards//mean": 0.7607421875,
+      "rewards//std": 0.02111050859093666,
+      "step": 514
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.103,
+      "grad_norm": 0.6811221241950989,
+      "kl": 0.2887964155524969,
+      "learning_rate": 4.8923796239449206e-06,
+      "loss": 0.0116,
+      "num_tokens": 4452008.0,
+      "reward": 0.76751708984375,
+      "reward_std": 0.006497236434370279,
+      "rewards//mean": 0.76751708984375,
+      "rewards//std": 0.029380397871136665,
+      "step": 515
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1032,
+      "grad_norm": 0.5849115252494812,
+      "kl": 0.2598783317953348,
+      "learning_rate": 4.891918618434305e-06,
+      "loss": 0.0104,
+      "num_tokens": 4460584.0,
+      "reward": 0.7503662109375,
+      "reward_std": 0.006470596417784691,
+      "rewards//mean": 0.7503662109375,
+      "rewards//std": 0.020818432793021202,
+      "step": 516
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1034,
+      "grad_norm": 0.774935245513916,
+      "kl": 0.2854543272405863,
+      "learning_rate": 4.891456649459156e-06,
+      "loss": 0.0114,
+      "num_tokens": 4469208.0,
+      "reward": 0.76519775390625,
+      "reward_std": 0.008341135457158089,
+      "rewards//mean": 0.76519775390625,
+      "rewards//std": 0.025450101122260094,
+      "step": 517
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1036,
+      "grad_norm": 0.7881346940994263,
+      "kl": 0.3002116158604622,
+      "learning_rate": 4.890993717205553e-06,
+      "loss": 0.012,
+      "num_tokens": 4477880.0,
+      "reward": 0.7275390625,
+      "reward_std": 0.007611682638525963,
+      "rewards//mean": 0.7275390625,
+      "rewards//std": 0.026013750582933426,
+      "step": 518
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1038,
+      "grad_norm": 0.7925445437431335,
+      "kl": 0.2746951151639223,
+      "learning_rate": 4.8905298218599685e-06,
+      "loss": 0.011,
+      "num_tokens": 4486520.0,
+      "reward": 0.753173828125,
+      "reward_std": 0.00768206175416708,
+      "rewards//mean": 0.753173828125,
+      "rewards//std": 0.03539228066802025,
+      "step": 519
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.104,
+      "grad_norm": 0.649895191192627,
+      "kl": 0.3011128604412079,
+      "learning_rate": 4.8900649636092565e-06,
+      "loss": 0.012,
+      "num_tokens": 4495064.0,
+      "reward": 0.76141357421875,
+      "reward_std": 0.008288520388305187,
+      "rewards//mean": 0.76141357421875,
+      "rewards//std": 0.029034705832600594,
+      "step": 520
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1042,
+      "grad_norm": 0.6826982498168945,
+      "kl": 0.27436352148652077,
+      "learning_rate": 4.889599142640663e-06,
+      "loss": 0.011,
+      "num_tokens": 4503728.0,
+      "reward": 0.73187255859375,
+      "reward_std": 0.007811560295522213,
+      "rewards//mean": 0.73187255859375,
+      "rewards//std": 0.031737327575683594,
+      "step": 521
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1044,
+      "grad_norm": 0.7717922329902649,
+      "kl": 0.2864828519523144,
+      "learning_rate": 4.889132359141822e-06,
+      "loss": 0.0115,
+      "num_tokens": 4512344.0,
+      "reward": 0.76220703125,
+      "reward_std": 0.005440828390419483,
+      "rewards//mean": 0.76220703125,
+      "rewards//std": 0.02468077465891838,
+      "step": 522
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1046,
+      "grad_norm": 0.6896048784255981,
+      "kl": 0.27172659896314144,
+      "learning_rate": 4.888664613300751e-06,
+      "loss": 0.0109,
+      "num_tokens": 4520896.0,
+      "reward": 0.74566650390625,
+      "reward_std": 0.008871195837855339,
+      "rewards//mean": 0.74566650390625,
+      "rewards//std": 0.02351093478500843,
+      "step": 523
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1048,
+      "grad_norm": 0.7394423484802246,
+      "kl": 0.2659613937139511,
+      "learning_rate": 4.888195905305859e-06,
+      "loss": 0.0106,
+      "num_tokens": 4529480.0,
+      "reward": 0.73236083984375,
+      "reward_std": 0.012172890827059746,
+      "rewards//mean": 0.73236083984375,
+      "rewards//std": 0.03177260234951973,
+      "step": 524
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.105,
+      "grad_norm": 0.6426916122436523,
+      "kl": 0.27845061011612415,
+      "learning_rate": 4.887726235345943e-06,
+      "loss": 0.0111,
+      "num_tokens": 4538064.0,
+      "reward": 0.76239013671875,
+      "reward_std": 0.008901185356080532,
+      "rewards//mean": 0.76239013671875,
+      "rewards//std": 0.023638714104890823,
+      "step": 525
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1052,
+      "grad_norm": 0.6519190669059753,
+      "kl": 0.25384664349257946,
+      "learning_rate": 4.8872556036101845e-06,
+      "loss": 0.0102,
+      "num_tokens": 4546688.0,
+      "reward": 0.77862548828125,
+      "reward_std": 0.009897831827402115,
+      "rewards//mean": 0.77862548828125,
+      "rewards//std": 0.029281822964549065,
+      "step": 526
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1054,
+      "grad_norm": 0.7181262969970703,
+      "kl": 0.26998190581798553,
+      "learning_rate": 4.886784010288155e-06,
+      "loss": 0.0108,
+      "num_tokens": 4555432.0,
+      "reward": 0.78851318359375,
+      "reward_std": 0.011629641987383366,
+      "rewards//mean": 0.78851318359375,
+      "rewards//std": 0.025476258248090744,
+      "step": 527
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1056,
+      "grad_norm": 0.6040878295898438,
+      "kl": 0.28468160331249237,
+      "learning_rate": 4.886311455569811e-06,
+      "loss": 0.0114,
+      "num_tokens": 4564064.0,
+      "reward": 0.76861572265625,
+      "reward_std": 0.0060232048854231834,
+      "rewards//mean": 0.76861572265625,
+      "rewards//std": 0.024147462099790573,
+      "step": 528
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1058,
+      "grad_norm": 0.6055912375450134,
+      "kl": 0.2735178656876087,
+      "learning_rate": 4.885837939645499e-06,
+      "loss": 0.0109,
+      "num_tokens": 4572752.0,
+      "reward": 0.75860595703125,
+      "reward_std": 0.005819528363645077,
+      "rewards//mean": 0.75860595703125,
+      "rewards//std": 0.025687508285045624,
+      "step": 529
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.106,
+      "grad_norm": 0.7213678956031799,
+      "kl": 0.2826583944261074,
+      "learning_rate": 4.885363462705949e-06,
+      "loss": 0.0113,
+      "num_tokens": 4581432.0,
+      "reward": 0.7806396484375,
+      "reward_std": 0.005271364934742451,
+      "rewards//mean": 0.7806396484375,
+      "rewards//std": 0.024071883410215378,
+      "step": 530
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1062,
+      "grad_norm": 0.6324445605278015,
+      "kl": 0.30165957659482956,
+      "learning_rate": 4.884888024942282e-06,
+      "loss": 0.0121,
+      "num_tokens": 4590056.0,
+      "reward": 0.7506103515625,
+      "reward_std": 0.0039962828159332275,
+      "rewards//mean": 0.7506103515625,
+      "rewards//std": 0.029759034514427185,
+      "step": 531
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1064,
+      "grad_norm": 0.6357660293579102,
+      "kl": 0.2692461237311363,
+      "learning_rate": 4.884411626546004e-06,
+      "loss": 0.0108,
+      "num_tokens": 4598776.0,
+      "reward": 0.73028564453125,
+      "reward_std": 0.006139049306511879,
+      "rewards//mean": 0.73028564453125,
+      "rewards//std": 0.020942674949765205,
+      "step": 532
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1066,
+      "grad_norm": 0.9847143292427063,
+      "kl": 0.30338218063116074,
+      "learning_rate": 4.883934267709007e-06,
+      "loss": 0.0121,
+      "num_tokens": 4607464.0,
+      "reward": 0.77239990234375,
+      "reward_std": 0.005770155228674412,
+      "rewards//mean": 0.77239990234375,
+      "rewards//std": 0.030767623335123062,
+      "step": 533
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1068,
+      "grad_norm": 0.8446683883666992,
+      "kl": 0.3181992806494236,
+      "learning_rate": 4.883455948623574e-06,
+      "loss": 0.0127,
+      "num_tokens": 4616104.0,
+      "reward": 0.7333984375,
+      "reward_std": 0.009834567084908485,
+      "rewards//mean": 0.7333984375,
+      "rewards//std": 0.032324355095624924,
+      "step": 534
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.107,
+      "grad_norm": 0.7224143743515015,
+      "kl": 0.2985565960407257,
+      "learning_rate": 4.882976669482368e-06,
+      "loss": 0.0119,
+      "num_tokens": 4624760.0,
+      "reward": 0.7509765625,
+      "reward_std": 0.006324178539216518,
+      "rewards//mean": 0.7509765625,
+      "rewards//std": 0.030273688957095146,
+      "step": 535
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1072,
+      "grad_norm": 0.6489189863204956,
+      "kl": 0.2836180441081524,
+      "learning_rate": 4.882496430478445e-06,
+      "loss": 0.0113,
+      "num_tokens": 4633392.0,
+      "reward": 0.7471923828125,
+      "reward_std": 0.00982951931655407,
+      "rewards//mean": 0.7471923828125,
+      "rewards//std": 0.027405409142374992,
+      "step": 536
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1074,
+      "grad_norm": 0.711584210395813,
+      "kl": 0.2708862889558077,
+      "learning_rate": 4.882015231805245e-06,
+      "loss": 0.0108,
+      "num_tokens": 4642072.0,
+      "reward": 0.7578125,
+      "reward_std": 0.008214281871914864,
+      "rewards//mean": 0.7578125,
+      "rewards//std": 0.027891865000128746,
+      "step": 537
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1076,
+      "grad_norm": 0.6661033034324646,
+      "kl": 0.26852802373468876,
+      "learning_rate": 4.881533073656594e-06,
+      "loss": 0.0107,
+      "num_tokens": 4650720.0,
+      "reward": 0.75445556640625,
+      "reward_std": 0.007886864244937897,
+      "rewards//mean": 0.75445556640625,
+      "rewards//std": 0.028012022376060486,
+      "step": 538
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1078,
+      "grad_norm": 0.7064325213432312,
+      "kl": 0.2860189266502857,
+      "learning_rate": 4.8810499562267066e-06,
+      "loss": 0.0114,
+      "num_tokens": 4659312.0,
+      "reward": 0.769287109375,
+      "reward_std": 0.006793664768338203,
+      "rewards//mean": 0.769287109375,
+      "rewards//std": 0.02851766347885132,
+      "step": 539
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.108,
+      "grad_norm": 0.6437175869941711,
+      "kl": 0.25416005309671164,
+      "learning_rate": 4.88056587971018e-06,
+      "loss": 0.0102,
+      "num_tokens": 4667896.0,
+      "reward": 0.7423095703125,
+      "reward_std": 0.007785624824464321,
+      "rewards//mean": 0.7423095703125,
+      "rewards//std": 0.028713131323456764,
+      "step": 540
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1082,
+      "grad_norm": 0.5998038053512573,
+      "kl": 0.2681788485497236,
+      "learning_rate": 4.880080844302004e-06,
+      "loss": 0.0107,
+      "num_tokens": 4676488.0,
+      "reward": 0.75537109375,
+      "reward_std": 0.006692672614008188,
+      "rewards//mean": 0.75537109375,
+      "rewards//std": 0.020033232867717743,
+      "step": 541
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1084,
+      "grad_norm": 0.6596677899360657,
+      "kl": 0.24427864141762257,
+      "learning_rate": 4.879594850197548e-06,
+      "loss": 0.0098,
+      "num_tokens": 4685128.0,
+      "reward": 0.7476806640625,
+      "reward_std": 0.010514110326766968,
+      "rewards//mean": 0.7476806640625,
+      "rewards//std": 0.03853638097643852,
+      "step": 542
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1086,
+      "grad_norm": 0.6608676910400391,
+      "kl": 0.259847916662693,
+      "learning_rate": 4.87910789759257e-06,
+      "loss": 0.0104,
+      "num_tokens": 4693824.0,
+      "reward": 0.75299072265625,
+      "reward_std": 0.005296648014336824,
+      "rewards//mean": 0.75299072265625,
+      "rewards//std": 0.048891909420490265,
+      "step": 543
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1088,
+      "grad_norm": 0.6944065093994141,
+      "kl": 0.2824724651873112,
+      "learning_rate": 4.878619986683215e-06,
+      "loss": 0.0113,
+      "num_tokens": 4702472.0,
+      "reward": 0.750244140625,
+      "reward_std": 0.007567228749394417,
+      "rewards//mean": 0.750244140625,
+      "rewards//std": 0.03861897811293602,
+      "step": 544
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.109,
+      "grad_norm": 0.6676400303840637,
+      "kl": 0.26609333604574203,
+      "learning_rate": 4.8781311176660144e-06,
+      "loss": 0.0106,
+      "num_tokens": 4711104.0,
+      "reward": 0.779052734375,
+      "reward_std": 0.010382162407040596,
+      "rewards//mean": 0.779052734375,
+      "rewards//std": 0.02472366951406002,
+      "step": 545
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1092,
+      "grad_norm": 0.6957513689994812,
+      "kl": 0.2683636359870434,
+      "learning_rate": 4.8776412907378845e-06,
+      "loss": 0.0107,
+      "num_tokens": 4719688.0,
+      "reward": 0.71490478515625,
+      "reward_std": 0.008024358190596104,
+      "rewards//mean": 0.71490478515625,
+      "rewards//std": 0.037217672914266586,
+      "step": 546
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1094,
+      "grad_norm": 0.6461659073829651,
+      "kl": 0.2612381912767887,
+      "learning_rate": 4.877150506096127e-06,
+      "loss": 0.0104,
+      "num_tokens": 4728272.0,
+      "reward": 0.7481689453125,
+      "reward_std": 0.006405083928257227,
+      "rewards//mean": 0.7481689453125,
+      "rewards//std": 0.02294633351266384,
+      "step": 547
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1096,
+      "grad_norm": 0.6994146704673767,
+      "kl": 0.2628567796200514,
+      "learning_rate": 4.8766587639384285e-06,
+      "loss": 0.0105,
+      "num_tokens": 4736888.0,
+      "reward": 0.74835205078125,
+      "reward_std": 0.006025420036166906,
+      "rewards//mean": 0.74835205078125,
+      "rewards//std": 0.028795981779694557,
+      "step": 548
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1098,
+      "grad_norm": 0.5853991508483887,
+      "kl": 0.25532799772918224,
+      "learning_rate": 4.876166064462866e-06,
+      "loss": 0.0102,
+      "num_tokens": 4745456.0,
+      "reward": 0.7703857421875,
+      "reward_std": 0.009814901277422905,
+      "rewards//mean": 0.7703857421875,
+      "rewards//std": 0.02585000917315483,
+      "step": 549
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.11,
+      "grad_norm": 0.6331080198287964,
+      "kl": 0.2674791272729635,
+      "learning_rate": 4.8756724078678955e-06,
+      "loss": 0.0107,
+      "num_tokens": 4754000.0,
+      "reward": 0.7891845703125,
+      "reward_std": 0.01081712357699871,
+      "rewards//mean": 0.7891845703125,
+      "rewards//std": 0.023632710799574852,
+      "step": 550
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1102,
+      "grad_norm": 0.789883553981781,
+      "kl": 0.259792210534215,
+      "learning_rate": 4.875177794352364e-06,
+      "loss": 0.0104,
+      "num_tokens": 4762808.0,
+      "reward": 0.7608642578125,
+      "reward_std": 0.006845149677246809,
+      "rewards//mean": 0.7608642578125,
+      "rewards//std": 0.0270137470215559,
+      "step": 551
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1104,
+      "grad_norm": 0.6203144788742065,
+      "kl": 0.2684528976678848,
+      "learning_rate": 4.8746822241155006e-06,
+      "loss": 0.0107,
+      "num_tokens": 4771488.0,
+      "reward": 0.7344970703125,
+      "reward_std": 0.007759134750813246,
+      "rewards//mean": 0.7344970703125,
+      "rewards//std": 0.02165386639535427,
+      "step": 552
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1106,
+      "grad_norm": 0.8759577870368958,
+      "kl": 0.2989007495343685,
+      "learning_rate": 4.874185697356921e-06,
+      "loss": 0.012,
+      "num_tokens": 4780144.0,
+      "reward": 0.7337646484375,
+      "reward_std": 0.009506863541901112,
+      "rewards//mean": 0.7337646484375,
+      "rewards//std": 0.02997797727584839,
+      "step": 553
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1108,
+      "grad_norm": 0.5836766958236694,
+      "kl": 0.25084237568080425,
+      "learning_rate": 4.873688214276628e-06,
+      "loss": 0.01,
+      "num_tokens": 4788744.0,
+      "reward": 0.71270751953125,
+      "reward_std": 0.008568651042878628,
+      "rewards//mean": 0.71270751953125,
+      "rewards//std": 0.03878828510642052,
+      "step": 554
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.111,
+      "grad_norm": 0.6924554109573364,
+      "kl": 0.2806801497936249,
+      "learning_rate": 4.873189775075005e-06,
+      "loss": 0.0112,
+      "num_tokens": 4797400.0,
+      "reward": 0.734375,
+      "reward_std": 0.0058593968860805035,
+      "rewards//mean": 0.734375,
+      "rewards//std": 0.03078145906329155,
+      "step": 555
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1112,
+      "grad_norm": 0.7575480341911316,
+      "kl": 0.2851990181952715,
+      "learning_rate": 4.872690379952824e-06,
+      "loss": 0.0114,
+      "num_tokens": 4806144.0,
+      "reward": 0.79217529296875,
+      "reward_std": 0.008231771178543568,
+      "rewards//mean": 0.79217529296875,
+      "rewards//std": 0.02772795408964157,
+      "step": 556
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1114,
+      "grad_norm": 0.625200092792511,
+      "kl": 0.26935046166181564,
+      "learning_rate": 4.8721900291112415e-06,
+      "loss": 0.0108,
+      "num_tokens": 4814792.0,
+      "reward": 0.762939453125,
+      "reward_std": 0.00905733834952116,
+      "rewards//mean": 0.762939453125,
+      "rewards//std": 0.03055935725569725,
+      "step": 557
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1116,
+      "grad_norm": 1.0318313837051392,
+      "kl": 0.31965751200914383,
+      "learning_rate": 4.871688722751799e-06,
+      "loss": 0.0128,
+      "num_tokens": 4823488.0,
+      "reward": 0.77117919921875,
+      "reward_std": 0.008040939457714558,
+      "rewards//mean": 0.77117919921875,
+      "rewards//std": 0.032330382615327835,
+      "step": 558
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1118,
+      "grad_norm": 0.6006855964660645,
+      "kl": 0.24019759241491556,
+      "learning_rate": 4.8711864610764235e-06,
+      "loss": 0.0096,
+      "num_tokens": 4832160.0,
+      "reward": 0.76800537109375,
+      "reward_std": 0.00946258008480072,
+      "rewards//mean": 0.76800537109375,
+      "rewards//std": 0.03378697484731674,
+      "step": 559
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.112,
+      "grad_norm": 0.7878683805465698,
+      "kl": 0.2661712933331728,
+      "learning_rate": 4.870683244287425e-06,
+      "loss": 0.0106,
+      "num_tokens": 4840776.0,
+      "reward": 0.720458984375,
+      "reward_std": 0.008840283378958702,
+      "rewards//mean": 0.720458984375,
+      "rewards//std": 0.04082028940320015,
+      "step": 560
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1122,
+      "grad_norm": 0.83661288022995,
+      "kl": 0.2538266107439995,
+      "learning_rate": 4.870179072587499e-06,
+      "loss": 0.0102,
+      "num_tokens": 4849480.0,
+      "reward": 0.7720947265625,
+      "reward_std": 0.009472770616412163,
+      "rewards//mean": 0.7720947265625,
+      "rewards//std": 0.025621790438890457,
+      "step": 561
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1124,
+      "grad_norm": 0.7142016887664795,
+      "kl": 0.28109513595700264,
+      "learning_rate": 4.869673946179726e-06,
+      "loss": 0.0112,
+      "num_tokens": 4858184.0,
+      "reward": 0.764404296875,
+      "reward_std": 0.00784839503467083,
+      "rewards//mean": 0.764404296875,
+      "rewards//std": 0.02929687686264515,
+      "step": 562
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1126,
+      "grad_norm": 0.7756282091140747,
+      "kl": 0.27597307227551937,
+      "learning_rate": 4.8691678652675715e-06,
+      "loss": 0.011,
+      "num_tokens": 4866856.0,
+      "reward": 0.76470947265625,
+      "reward_std": 0.008997954428195953,
+      "rewards//mean": 0.76470947265625,
+      "rewards//std": 0.02482014335691929,
+      "step": 563
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1128,
+      "grad_norm": 0.6697260141372681,
+      "kl": 0.25615566223859787,
+      "learning_rate": 4.8686608300548836e-06,
+      "loss": 0.0102,
+      "num_tokens": 4875376.0,
+      "reward": 0.732421875,
+      "reward_std": 0.007549830712378025,
+      "rewards//mean": 0.732421875,
+      "rewards//std": 0.030193578451871872,
+      "step": 564
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.113,
+      "grad_norm": 0.7746402621269226,
+      "kl": 0.2652531825006008,
+      "learning_rate": 4.868152840745896e-06,
+      "loss": 0.0106,
+      "num_tokens": 4884008.0,
+      "reward": 0.7220458984375,
+      "reward_std": 0.00881132297217846,
+      "rewards//mean": 0.7220458984375,
+      "rewards//std": 0.036915309727191925,
+      "step": 565
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1132,
+      "grad_norm": 0.7145909667015076,
+      "kl": 0.2629079818725586,
+      "learning_rate": 4.8676438975452276e-06,
+      "loss": 0.0105,
+      "num_tokens": 4892672.0,
+      "reward": 0.7686767578125,
+      "reward_std": 0.008167712949216366,
+      "rewards//mean": 0.7686767578125,
+      "rewards//std": 0.02817239984869957,
+      "step": 566
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1134,
+      "grad_norm": 0.7171170711517334,
+      "kl": 0.22697240114212036,
+      "learning_rate": 4.86713400065788e-06,
+      "loss": 0.0091,
+      "num_tokens": 4901272.0,
+      "reward": 0.7269287109375,
+      "reward_std": 0.009677172638475895,
+      "rewards//mean": 0.7269287109375,
+      "rewards//std": 0.038479775190353394,
+      "step": 567
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1136,
+      "grad_norm": 0.6745283603668213,
+      "kl": 0.24715841561555862,
+      "learning_rate": 4.866623150289241e-06,
+      "loss": 0.0099,
+      "num_tokens": 4909840.0,
+      "reward": 0.75732421875,
+      "reward_std": 0.008682534098625183,
+      "rewards//mean": 0.75732421875,
+      "rewards//std": 0.024094825610518456,
+      "step": 568
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1138,
+      "grad_norm": 0.6382898688316345,
+      "kl": 0.25850758142769337,
+      "learning_rate": 4.86611134664508e-06,
+      "loss": 0.0103,
+      "num_tokens": 4918384.0,
+      "reward": 0.7513427734375,
+      "reward_std": 0.0103992260992527,
+      "rewards//mean": 0.7513427734375,
+      "rewards//std": 0.020102253183722496,
+      "step": 569
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.114,
+      "grad_norm": 0.9113550186157227,
+      "kl": 0.25279413163661957,
+      "learning_rate": 4.865598589931552e-06,
+      "loss": 0.0101,
+      "num_tokens": 4927096.0,
+      "reward": 0.7747802734375,
+      "reward_std": 0.007576893083751202,
+      "rewards//mean": 0.7747802734375,
+      "rewards//std": 0.033690787851810455,
+      "step": 570
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1142,
+      "grad_norm": 0.6302274465560913,
+      "kl": 0.25698636658489704,
+      "learning_rate": 4.865084880355193e-06,
+      "loss": 0.0103,
+      "num_tokens": 4935776.0,
+      "reward": 0.73907470703125,
+      "reward_std": 0.007312727626413107,
+      "rewards//mean": 0.73907470703125,
+      "rewards//std": 0.026749854907393456,
+      "step": 571
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1144,
+      "grad_norm": 0.5283216834068298,
+      "kl": 0.22627251036465168,
+      "learning_rate": 4.864570218122928e-06,
+      "loss": 0.0091,
+      "num_tokens": 4944400.0,
+      "reward": 0.73974609375,
+      "reward_std": 0.01004891935735941,
+      "rewards//mean": 0.73974609375,
+      "rewards//std": 0.02998024970293045,
+      "step": 572
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1146,
+      "grad_norm": 0.6809362769126892,
+      "kl": 0.24691984988749027,
+      "learning_rate": 4.864054603442063e-06,
+      "loss": 0.0099,
+      "num_tokens": 4953000.0,
+      "reward": 0.75897216796875,
+      "reward_std": 0.009808078408241272,
+      "rewards//mean": 0.75897216796875,
+      "rewards//std": 0.025493483990430832,
+      "step": 573
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1148,
+      "grad_norm": 0.6428877711296082,
+      "kl": 0.23521161265671253,
+      "learning_rate": 4.863538036520285e-06,
+      "loss": 0.0094,
+      "num_tokens": 4961696.0,
+      "reward": 0.7613525390625,
+      "reward_std": 0.010448819026350975,
+      "rewards//mean": 0.7613525390625,
+      "rewards//std": 0.03130976855754852,
+      "step": 574
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.115,
+      "grad_norm": 0.6935397982597351,
+      "kl": 0.23570332117378712,
+      "learning_rate": 4.863020517565669e-06,
+      "loss": 0.0094,
+      "num_tokens": 4970296.0,
+      "reward": 0.743896484375,
+      "reward_std": 0.007338499650359154,
+      "rewards//mean": 0.743896484375,
+      "rewards//std": 0.024899380281567574,
+      "step": 575
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1152,
+      "grad_norm": 0.6086986660957336,
+      "kl": 0.2525392398238182,
+      "learning_rate": 4.862502046786671e-06,
+      "loss": 0.0101,
+      "num_tokens": 4979040.0,
+      "reward": 0.748779296875,
+      "reward_std": 0.007959538139402866,
+      "rewards//mean": 0.748779296875,
+      "rewards//std": 0.04012007638812065,
+      "step": 576
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1154,
+      "grad_norm": 0.6362279653549194,
+      "kl": 0.21946440637111664,
+      "learning_rate": 4.861982624392132e-06,
+      "loss": 0.0088,
+      "num_tokens": 4987720.0,
+      "reward": 0.78765869140625,
+      "reward_std": 0.009038891643285751,
+      "rewards//mean": 0.78765869140625,
+      "rewards//std": 0.020556749776005745,
+      "step": 577
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1156,
+      "grad_norm": 0.6334317326545715,
+      "kl": 0.2258777841925621,
+      "learning_rate": 4.861462250591273e-06,
+      "loss": 0.009,
+      "num_tokens": 4996352.0,
+      "reward": 0.719482421875,
+      "reward_std": 0.009759314358234406,
+      "rewards//mean": 0.719482421875,
+      "rewards//std": 0.03258461877703667,
+      "step": 578
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1158,
+      "grad_norm": 0.585574746131897,
+      "kl": 0.2085122261196375,
+      "learning_rate": 4.860940925593703e-06,
+      "loss": 0.0083,
+      "num_tokens": 5005168.0,
+      "reward": 0.7581787109375,
+      "reward_std": 0.010835596360266209,
+      "rewards//mean": 0.7581787109375,
+      "rewards//std": 0.03846088796854019,
+      "step": 579
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.116,
+      "grad_norm": 0.6512745022773743,
+      "kl": 0.2308096420019865,
+      "learning_rate": 4.86041864960941e-06,
+      "loss": 0.0092,
+      "num_tokens": 5013744.0,
+      "reward": 0.75238037109375,
+      "reward_std": 0.008994483388960361,
+      "rewards//mean": 0.75238037109375,
+      "rewards//std": 0.026278842240571976,
+      "step": 580
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1162,
+      "grad_norm": 0.5345540642738342,
+      "kl": 0.2112738210707903,
+      "learning_rate": 4.859895422848767e-06,
+      "loss": 0.0085,
+      "num_tokens": 5022304.0,
+      "reward": 0.73822021484375,
+      "reward_std": 0.008295232430100441,
+      "rewards//mean": 0.73822021484375,
+      "rewards//std": 0.04076458141207695,
+      "step": 581
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1164,
+      "grad_norm": 0.602761447429657,
+      "kl": 0.21912161633372307,
+      "learning_rate": 4.859371245522531e-06,
+      "loss": 0.0088,
+      "num_tokens": 5030944.0,
+      "reward": 0.703369140625,
+      "reward_std": 0.00949187483638525,
+      "rewards//mean": 0.703369140625,
+      "rewards//std": 0.028560098260641098,
+      "step": 582
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1166,
+      "grad_norm": 0.6504238247871399,
+      "kl": 0.23803219199180603,
+      "learning_rate": 4.8588461178418375e-06,
+      "loss": 0.0095,
+      "num_tokens": 5039632.0,
+      "reward": 0.7572021484375,
+      "reward_std": 0.00934354867786169,
+      "rewards//mean": 0.7572021484375,
+      "rewards//std": 0.02799776755273342,
+      "step": 583
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1168,
+      "grad_norm": 0.5838614702224731,
+      "kl": 0.21050356701016426,
+      "learning_rate": 4.858320040018212e-06,
+      "loss": 0.0084,
+      "num_tokens": 5048256.0,
+      "reward": 0.75494384765625,
+      "reward_std": 0.009613383561372757,
+      "rewards//mean": 0.75494384765625,
+      "rewards//std": 0.021341485902667046,
+      "step": 584
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.117,
+      "grad_norm": 0.5332762002944946,
+      "kl": 0.2162165530025959,
+      "learning_rate": 4.857793012263555e-06,
+      "loss": 0.0086,
+      "num_tokens": 5056808.0,
+      "reward": 0.75299072265625,
+      "reward_std": 0.00866914913058281,
+      "rewards//mean": 0.75299072265625,
+      "rewards//std": 0.030653268098831177,
+      "step": 585
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1172,
+      "grad_norm": 0.5869634747505188,
+      "kl": 0.22080680169165134,
+      "learning_rate": 4.857265034790155e-06,
+      "loss": 0.0088,
+      "num_tokens": 5065400.0,
+      "reward": 0.7698974609375,
+      "reward_std": 0.009216565638780594,
+      "rewards//mean": 0.7698974609375,
+      "rewards//std": 0.03380203619599342,
+      "step": 586
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1174,
+      "grad_norm": 0.6322459578514099,
+      "kl": 0.20717273838818073,
+      "learning_rate": 4.85673610781068e-06,
+      "loss": 0.0083,
+      "num_tokens": 5074064.0,
+      "reward": 0.78057861328125,
+      "reward_std": 0.010938020423054695,
+      "rewards//mean": 0.78057861328125,
+      "rewards//std": 0.028258977457880974,
+      "step": 587
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1176,
+      "grad_norm": 0.8341087102890015,
+      "kl": 0.2445808406919241,
+      "learning_rate": 4.856206231538184e-06,
+      "loss": 0.0098,
+      "num_tokens": 5082720.0,
+      "reward": 0.74163818359375,
+      "reward_std": 0.009593227878212929,
+      "rewards//mean": 0.74163818359375,
+      "rewards//std": 0.02997513860464096,
+      "step": 588
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1178,
+      "grad_norm": 0.5464284420013428,
+      "kl": 0.20626290701329708,
+      "learning_rate": 4.855675406186099e-06,
+      "loss": 0.0083,
+      "num_tokens": 5091344.0,
+      "reward": 0.73687744140625,
+      "reward_std": 0.013462206348776817,
+      "rewards//mean": 0.73687744140625,
+      "rewards//std": 0.03030860796570778,
+      "step": 589
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.118,
+      "grad_norm": 0.5974122285842896,
+      "kl": 0.2238724958151579,
+      "learning_rate": 4.855143631968242e-06,
+      "loss": 0.009,
+      "num_tokens": 5100024.0,
+      "reward": 0.76165771484375,
+      "reward_std": 0.010216079652309418,
+      "rewards//mean": 0.76165771484375,
+      "rewards//std": 0.022633064538240433,
+      "step": 590
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1182,
+      "grad_norm": 0.5498731732368469,
+      "kl": 0.20982307940721512,
+      "learning_rate": 4.854610909098813e-06,
+      "loss": 0.0084,
+      "num_tokens": 5108640.0,
+      "reward": 0.740234375,
+      "reward_std": 0.011146800592541695,
+      "rewards//mean": 0.740234375,
+      "rewards//std": 0.029065513983368874,
+      "step": 591
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1184,
+      "grad_norm": 0.5892578363418579,
+      "kl": 0.22463841922581196,
+      "learning_rate": 4.854077237792389e-06,
+      "loss": 0.009,
+      "num_tokens": 5117224.0,
+      "reward": 0.74560546875,
+      "reward_std": 0.010708848014473915,
+      "rewards//mean": 0.74560546875,
+      "rewards//std": 0.03500435873866081,
+      "step": 592
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1186,
+      "grad_norm": 0.5446705222129822,
+      "kl": 0.19729456305503845,
+      "learning_rate": 4.853542618263937e-06,
+      "loss": 0.0079,
+      "num_tokens": 5125776.0,
+      "reward": 0.7720947265625,
+      "reward_std": 0.0137183777987957,
+      "rewards//mean": 0.7720947265625,
+      "rewards//std": 0.02645888738334179,
+      "step": 593
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1188,
+      "grad_norm": 0.5078734159469604,
+      "kl": 0.20582310110330582,
+      "learning_rate": 4.8530070507288e-06,
+      "loss": 0.0082,
+      "num_tokens": 5134408.0,
+      "reward": 0.7685546875,
+      "reward_std": 0.008237513713538647,
+      "rewards//mean": 0.7685546875,
+      "rewards//std": 0.02214088849723339,
+      "step": 594
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.119,
+      "grad_norm": 0.6351189017295837,
+      "kl": 0.2431877087801695,
+      "learning_rate": 4.852470535402703e-06,
+      "loss": 0.0097,
+      "num_tokens": 5143040.0,
+      "reward": 0.75970458984375,
+      "reward_std": 0.009093962609767914,
+      "rewards//mean": 0.75970458984375,
+      "rewards//std": 0.021931972354650497,
+      "step": 595
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1192,
+      "grad_norm": 0.8271527886390686,
+      "kl": 0.23255185037851334,
+      "learning_rate": 4.851933072501756e-06,
+      "loss": 0.0093,
+      "num_tokens": 5151704.0,
+      "reward": 0.7572021484375,
+      "reward_std": 0.011184893548488617,
+      "rewards//mean": 0.7572021484375,
+      "rewards//std": 0.02380884252488613,
+      "step": 596
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1194,
+      "grad_norm": 0.7389485239982605,
+      "kl": 0.21711469255387783,
+      "learning_rate": 4.851394662242449e-06,
+      "loss": 0.0087,
+      "num_tokens": 5160400.0,
+      "reward": 0.7572021484375,
+      "reward_std": 0.012733031064271927,
+      "rewards//mean": 0.7572021484375,
+      "rewards//std": 0.027341259643435478,
+      "step": 597
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1196,
+      "grad_norm": 0.7461937665939331,
+      "kl": 0.2250336967408657,
+      "learning_rate": 4.850855304841653e-06,
+      "loss": 0.009,
+      "num_tokens": 5169024.0,
+      "reward": 0.75531005859375,
+      "reward_std": 0.006518941838294268,
+      "rewards//mean": 0.75531005859375,
+      "rewards//std": 0.027677135542035103,
+      "step": 598
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1198,
+      "grad_norm": 0.737876832485199,
+      "kl": 0.21269006468355656,
+      "learning_rate": 4.8503150005166225e-06,
+      "loss": 0.0085,
+      "num_tokens": 5177672.0,
+      "reward": 0.76031494140625,
+      "reward_std": 0.007318540476262569,
+      "rewards//mean": 0.76031494140625,
+      "rewards//std": 0.02775578200817108,
+      "step": 599
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.12,
+      "grad_norm": 0.6356746554374695,
+      "kl": 0.22319811396300793,
+      "learning_rate": 4.849773749484989e-06,
+      "loss": 0.0089,
+      "num_tokens": 5186400.0,
+      "reward": 0.72882080078125,
+      "reward_std": 0.008081294596195221,
+      "rewards//mean": 0.72882080078125,
+      "rewards//std": 0.03312958776950836,
+      "step": 600
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1202,
+      "grad_norm": 0.5195296406745911,
+      "kl": 0.20459580793976784,
+      "learning_rate": 4.849231551964771e-06,
+      "loss": 0.0082,
+      "num_tokens": 5195048.0,
+      "reward": 0.73486328125,
+      "reward_std": 0.009343027137219906,
+      "rewards//mean": 0.73486328125,
+      "rewards//std": 0.02382187359035015,
+      "step": 601
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1204,
+      "grad_norm": 0.6819605827331543,
+      "kl": 0.22957494854927063,
+      "learning_rate": 4.848688408174366e-06,
+      "loss": 0.0092,
+      "num_tokens": 5203712.0,
+      "reward": 0.7725830078125,
+      "reward_std": 0.008309951052069664,
+      "rewards//mean": 0.7725830078125,
+      "rewards//std": 0.02304638922214508,
+      "step": 602
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1206,
+      "grad_norm": 0.6028555631637573,
+      "kl": 0.2299717701971531,
+      "learning_rate": 4.84814431833255e-06,
+      "loss": 0.0092,
+      "num_tokens": 5212352.0,
+      "reward": 0.76513671875,
+      "reward_std": 0.011560057289898396,
+      "rewards//mean": 0.76513671875,
+      "rewards//std": 0.026951193809509277,
+      "step": 603
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1208,
+      "grad_norm": 0.7233465313911438,
+      "kl": 0.21515335515141487,
+      "learning_rate": 4.847599282658483e-06,
+      "loss": 0.0086,
+      "num_tokens": 5220912.0,
+      "reward": 0.7650146484375,
+      "reward_std": 0.009766367264091969,
+      "rewards//mean": 0.7650146484375,
+      "rewards//std": 0.03968513756990433,
+      "step": 604
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.121,
+      "grad_norm": 0.6206122636795044,
+      "kl": 0.2212205920368433,
+      "learning_rate": 4.847053301371706e-06,
+      "loss": 0.0088,
+      "num_tokens": 5229616.0,
+      "reward": 0.75567626953125,
+      "reward_std": 0.010858502238988876,
+      "rewards//mean": 0.75567626953125,
+      "rewards//std": 0.02769080549478531,
+      "step": 605
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1212,
+      "grad_norm": 0.6429035663604736,
+      "kl": 0.22523421607911587,
+      "learning_rate": 4.84650637469214e-06,
+      "loss": 0.009,
+      "num_tokens": 5238192.0,
+      "reward": 0.73785400390625,
+      "reward_std": 0.010889837518334389,
+      "rewards//mean": 0.73785400390625,
+      "rewards//std": 0.030354522168636322,
+      "step": 606
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1214,
+      "grad_norm": 0.7371296286582947,
+      "kl": 0.22557489946484566,
+      "learning_rate": 4.845958502840087e-06,
+      "loss": 0.009,
+      "num_tokens": 5246720.0,
+      "reward": 0.7720947265625,
+      "reward_std": 0.008845320902764797,
+      "rewards//mean": 0.7720947265625,
+      "rewards//std": 0.03540147468447685,
+      "step": 607
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1216,
+      "grad_norm": 0.8345296382904053,
+      "kl": 0.2478946428745985,
+      "learning_rate": 4.8454096860362284e-06,
+      "loss": 0.0099,
+      "num_tokens": 5255328.0,
+      "reward": 0.74542236328125,
+      "reward_std": 0.008497816510498524,
+      "rewards//mean": 0.74542236328125,
+      "rewards//std": 0.02872650697827339,
+      "step": 608
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1218,
+      "grad_norm": 0.9587209224700928,
+      "kl": 0.22490216977894306,
+      "learning_rate": 4.8448599245016306e-06,
+      "loss": 0.009,
+      "num_tokens": 5263992.0,
+      "reward": 0.7431640625,
+      "reward_std": 0.008303426206111908,
+      "rewards//mean": 0.7431640625,
+      "rewards//std": 0.033689215779304504,
+      "step": 609
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.122,
+      "grad_norm": 0.6855178475379944,
+      "kl": 0.24397451616823673,
+      "learning_rate": 4.844309218457735e-06,
+      "loss": 0.0098,
+      "num_tokens": 5272632.0,
+      "reward": 0.74456787109375,
+      "reward_std": 0.007743033580482006,
+      "rewards//mean": 0.74456787109375,
+      "rewards//std": 0.02703300304710865,
+      "step": 610
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1222,
+      "grad_norm": 0.8738477230072021,
+      "kl": 0.24380822479724884,
+      "learning_rate": 4.843757568126366e-06,
+      "loss": 0.0098,
+      "num_tokens": 5281264.0,
+      "reward": 0.76068115234375,
+      "reward_std": 0.008802594617009163,
+      "rewards//mean": 0.76068115234375,
+      "rewards//std": 0.02518342435359955,
+      "step": 611
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1224,
+      "grad_norm": 1.1533160209655762,
+      "kl": 0.2847021333873272,
+      "learning_rate": 4.84320497372973e-06,
+      "loss": 0.0114,
+      "num_tokens": 5290096.0,
+      "reward": 0.7354736328125,
+      "reward_std": 0.009391937404870987,
+      "rewards//mean": 0.7354736328125,
+      "rewards//std": 0.03238774463534355,
+      "step": 612
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1226,
+      "grad_norm": 0.9461967945098877,
+      "kl": 0.2901354692876339,
+      "learning_rate": 4.8426514354904096e-06,
+      "loss": 0.0116,
+      "num_tokens": 5298720.0,
+      "reward": 0.75830078125,
+      "reward_std": 0.01141531765460968,
+      "rewards//mean": 0.75830078125,
+      "rewards//std": 0.02832699380815029,
+      "step": 613
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1228,
+      "grad_norm": 0.8776152729988098,
+      "kl": 0.2779980804771185,
+      "learning_rate": 4.842096953631371e-06,
+      "loss": 0.0111,
+      "num_tokens": 5307344.0,
+      "reward": 0.74365234375,
+      "reward_std": 0.007914327085018158,
+      "rewards//mean": 0.74365234375,
+      "rewards//std": 0.03176122531294823,
+      "step": 614
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.123,
+      "grad_norm": 0.7875434160232544,
+      "kl": 0.27193955332040787,
+      "learning_rate": 4.841541528375961e-06,
+      "loss": 0.0109,
+      "num_tokens": 5315880.0,
+      "reward": 0.7650146484375,
+      "reward_std": 0.011012593284249306,
+      "rewards//mean": 0.7650146484375,
+      "rewards//std": 0.03217579424381256,
+      "step": 615
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1232,
+      "grad_norm": 0.9433616995811462,
+      "kl": 0.331559956073761,
+      "learning_rate": 4.840985159947902e-06,
+      "loss": 0.0133,
+      "num_tokens": 5324592.0,
+      "reward": 0.759521484375,
+      "reward_std": 0.008395890705287457,
+      "rewards//mean": 0.759521484375,
+      "rewards//std": 0.025189509615302086,
+      "step": 616
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1234,
+      "grad_norm": 1.2481539249420166,
+      "kl": 0.3108705338090658,
+      "learning_rate": 4.8404278485713005e-06,
+      "loss": 0.0124,
+      "num_tokens": 5333288.0,
+      "reward": 0.75555419921875,
+      "reward_std": 0.006895901635289192,
+      "rewards//mean": 0.75555419921875,
+      "rewards//std": 0.020772846415638924,
+      "step": 617
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1236,
+      "grad_norm": 1.051727056503296,
+      "kl": 0.3007947914302349,
+      "learning_rate": 4.839869594470642e-06,
+      "loss": 0.012,
+      "num_tokens": 5341944.0,
+      "reward": 0.74566650390625,
+      "reward_std": 0.009581143036484718,
+      "rewards//mean": 0.74566650390625,
+      "rewards//std": 0.02763991802930832,
+      "step": 618
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1238,
+      "grad_norm": 1.1710658073425293,
+      "kl": 0.3423069082200527,
+      "learning_rate": 4.839310397870791e-06,
+      "loss": 0.0137,
+      "num_tokens": 5350712.0,
+      "reward": 0.80609130859375,
+      "reward_std": 0.0071121640503406525,
+      "rewards//mean": 0.80609130859375,
+      "rewards//std": 0.022340906783938408,
+      "step": 619
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.124,
+      "grad_norm": 0.9958809018135071,
+      "kl": 0.30707372911274433,
+      "learning_rate": 4.838750258996992e-06,
+      "loss": 0.0123,
+      "num_tokens": 5359400.0,
+      "reward": 0.7572021484375,
+      "reward_std": 0.008860534057021141,
+      "rewards//mean": 0.7572021484375,
+      "rewards//std": 0.029439911246299744,
+      "step": 620
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1242,
+      "grad_norm": 1.331772804260254,
+      "kl": 0.3642958104610443,
+      "learning_rate": 4.838189178074867e-06,
+      "loss": 0.0146,
+      "num_tokens": 5368040.0,
+      "reward": 0.7486572265625,
+      "reward_std": 0.005241964012384415,
+      "rewards//mean": 0.7486572265625,
+      "rewards//std": 0.03716379404067993,
+      "step": 621
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1244,
+      "grad_norm": 1.1566888093948364,
+      "kl": 0.32168351113796234,
+      "learning_rate": 4.837627155330421e-06,
+      "loss": 0.0129,
+      "num_tokens": 5376656.0,
+      "reward": 0.74420166015625,
+      "reward_std": 0.010309340432286263,
+      "rewards//mean": 0.74420166015625,
+      "rewards//std": 0.035138972103595734,
+      "step": 622
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1246,
+      "grad_norm": 1.2128782272338867,
+      "kl": 0.3658299557864666,
+      "learning_rate": 4.837064190990036e-06,
+      "loss": 0.0146,
+      "num_tokens": 5385312.0,
+      "reward": 0.74615478515625,
+      "reward_std": 0.005186946596950293,
+      "rewards//mean": 0.74615478515625,
+      "rewards//std": 0.023983918130397797,
+      "step": 623
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1248,
+      "grad_norm": 1.2308634519577026,
+      "kl": 0.35844672471284866,
+      "learning_rate": 4.836500285280476e-06,
+      "loss": 0.0143,
+      "num_tokens": 5393888.0,
+      "reward": 0.75299072265625,
+      "reward_std": 0.0063725742511451244,
+      "rewards//mean": 0.75299072265625,
+      "rewards//std": 0.028277721256017685,
+      "step": 624
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.125,
+      "grad_norm": 1.1482653617858887,
+      "kl": 0.345749881118536,
+      "learning_rate": 4.83593543842888e-06,
+      "loss": 0.0138,
+      "num_tokens": 5402496.0,
+      "reward": 0.7623291015625,
+      "reward_std": 0.00810457207262516,
+      "rewards//mean": 0.7623291015625,
+      "rewards//std": 0.03183525428175926,
+      "step": 625
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1252,
+      "grad_norm": 1.252425193786621,
+      "kl": 0.4149610288441181,
+      "learning_rate": 4.835369650662767e-06,
+      "loss": 0.0166,
+      "num_tokens": 5411160.0,
+      "reward": 0.7589111328125,
+      "reward_std": 0.009770114906132221,
+      "rewards//mean": 0.7589111328125,
+      "rewards//std": 0.043982502073049545,
+      "step": 626
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1254,
+      "grad_norm": 1.0089774131774902,
+      "kl": 0.3325713388621807,
+      "learning_rate": 4.83480292221004e-06,
+      "loss": 0.0133,
+      "num_tokens": 5419800.0,
+      "reward": 0.78118896484375,
+      "reward_std": 0.007964782416820526,
+      "rewards//mean": 0.78118896484375,
+      "rewards//std": 0.027581803500652313,
+      "step": 627
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1256,
+      "grad_norm": 1.1872444152832031,
+      "kl": 0.3795311227440834,
+      "learning_rate": 4.834235253298973e-06,
+      "loss": 0.0152,
+      "num_tokens": 5428536.0,
+      "reward": 0.74444580078125,
+      "reward_std": 0.007206289563328028,
+      "rewards//mean": 0.74444580078125,
+      "rewards//std": 0.021763604134321213,
+      "step": 628
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1258,
+      "grad_norm": 1.326137661933899,
+      "kl": 0.37480510398745537,
+      "learning_rate": 4.833666644158227e-06,
+      "loss": 0.015,
+      "num_tokens": 5437144.0,
+      "reward": 0.7896728515625,
+      "reward_std": 0.00821404904127121,
+      "rewards//mean": 0.7896728515625,
+      "rewards//std": 0.026238271966576576,
+      "step": 629
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.126,
+      "grad_norm": 1.089011788368225,
+      "kl": 0.30840661004185677,
+      "learning_rate": 4.833097095016835e-06,
+      "loss": 0.0123,
+      "num_tokens": 5445856.0,
+      "reward": 0.76019287109375,
+      "reward_std": 0.008738137781620026,
+      "rewards//mean": 0.76019287109375,
+      "rewards//std": 0.028870007023215294,
+      "step": 630
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1262,
+      "grad_norm": 1.430968165397644,
+      "kl": 0.3591831102967262,
+      "learning_rate": 4.832526606104213e-06,
+      "loss": 0.0144,
+      "num_tokens": 5454512.0,
+      "reward": 0.732421875,
+      "reward_std": 0.008345866575837135,
+      "rewards//mean": 0.732421875,
+      "rewards//std": 0.032008104026317596,
+      "step": 631
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1264,
+      "grad_norm": 1.2066965103149414,
+      "kl": 0.35580387338995934,
+      "learning_rate": 4.831955177650153e-06,
+      "loss": 0.0142,
+      "num_tokens": 5463144.0,
+      "reward": 0.76519775390625,
+      "reward_std": 0.007233831100165844,
+      "rewards//mean": 0.76519775390625,
+      "rewards//std": 0.02703748270869255,
+      "step": 632
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1266,
+      "grad_norm": 1.33864426612854,
+      "kl": 0.3583483546972275,
+      "learning_rate": 4.831382809884826e-06,
+      "loss": 0.0143,
+      "num_tokens": 5471768.0,
+      "reward": 0.751708984375,
+      "reward_std": 0.006699239369481802,
+      "rewards//mean": 0.751708984375,
+      "rewards//std": 0.026021895930171013,
+      "step": 633
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1268,
+      "grad_norm": 1.0695289373397827,
+      "kl": 0.331091046333313,
+      "learning_rate": 4.830809503038781e-06,
+      "loss": 0.0132,
+      "num_tokens": 5480384.0,
+      "reward": 0.7537841796875,
+      "reward_std": 0.007534695789217949,
+      "rewards//mean": 0.7537841796875,
+      "rewards//std": 0.020481223240494728,
+      "step": 634
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.127,
+      "grad_norm": 1.05119788646698,
+      "kl": 0.2932460140436888,
+      "learning_rate": 4.830235257342948e-06,
+      "loss": 0.0117,
+      "num_tokens": 5488984.0,
+      "reward": 0.73675537109375,
+      "reward_std": 0.009710824117064476,
+      "rewards//mean": 0.73675537109375,
+      "rewards//std": 0.02311091497540474,
+      "step": 635
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1272,
+      "grad_norm": 0.9468859434127808,
+      "kl": 0.27140727639198303,
+      "learning_rate": 4.829660073028631e-06,
+      "loss": 0.0109,
+      "num_tokens": 5497624.0,
+      "reward": 0.81549072265625,
+      "reward_std": 0.006733256857842207,
+      "rewards//mean": 0.81549072265625,
+      "rewards//std": 0.017603833228349686,
+      "step": 636
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1274,
+      "grad_norm": 1.144340991973877,
+      "kl": 0.3296862170100212,
+      "learning_rate": 4.829083950327516e-06,
+      "loss": 0.0132,
+      "num_tokens": 5506312.0,
+      "reward": 0.7391357421875,
+      "reward_std": 0.007396392058581114,
+      "rewards//mean": 0.7391357421875,
+      "rewards//std": 0.02667086571455002,
+      "step": 637
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1276,
+      "grad_norm": 1.0231379270553589,
+      "kl": 0.32026514038443565,
+      "learning_rate": 4.828506889471664e-06,
+      "loss": 0.0128,
+      "num_tokens": 5514912.0,
+      "reward": 0.73388671875,
+      "reward_std": 0.006786069832742214,
+      "rewards//mean": 0.73388671875,
+      "rewards//std": 0.017696566879749298,
+      "step": 638
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1278,
+      "grad_norm": 1.3025801181793213,
+      "kl": 0.3653467670083046,
+      "learning_rate": 4.827928890693515e-06,
+      "loss": 0.0146,
+      "num_tokens": 5523536.0,
+      "reward": 0.782958984375,
+      "reward_std": 0.0090708639472723,
+      "rewards//mean": 0.782958984375,
+      "rewards//std": 0.0243386123329401,
+      "step": 639
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.128,
+      "grad_norm": 1.0168287754058838,
+      "kl": 0.3438634071499109,
+      "learning_rate": 4.8273499542258885e-06,
+      "loss": 0.0138,
+      "num_tokens": 5532184.0,
+      "reward": 0.7303466796875,
+      "reward_std": 0.007754259742796421,
+      "rewards//mean": 0.7303466796875,
+      "rewards//std": 0.027673648670315742,
+      "step": 640
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1282,
+      "grad_norm": 0.9684326648712158,
+      "kl": 0.3083877395838499,
+      "learning_rate": 4.826770080301978e-06,
+      "loss": 0.0123,
+      "num_tokens": 5540808.0,
+      "reward": 0.7808837890625,
+      "reward_std": 0.010280175134539604,
+      "rewards//mean": 0.7808837890625,
+      "rewards//std": 0.029550766572356224,
+      "step": 641
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1284,
+      "grad_norm": 1.223793864250183,
+      "kl": 0.34201204031705856,
+      "learning_rate": 4.826189269155357e-06,
+      "loss": 0.0137,
+      "num_tokens": 5549448.0,
+      "reward": 0.7252197265625,
+      "reward_std": 0.008540410548448563,
+      "rewards//mean": 0.7252197265625,
+      "rewards//std": 0.029250076040625572,
+      "step": 642
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1286,
+      "grad_norm": 1.2303458452224731,
+      "kl": 0.36477214470505714,
+      "learning_rate": 4.825607521019978e-06,
+      "loss": 0.0146,
+      "num_tokens": 5558080.0,
+      "reward": 0.7886962890625,
+      "reward_std": 0.00828670896589756,
+      "rewards//mean": 0.7886962890625,
+      "rewards//std": 0.022837886586785316,
+      "step": 643
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1288,
+      "grad_norm": 1.127670168876648,
+      "kl": 0.36717789620161057,
+      "learning_rate": 4.825024836130166e-06,
+      "loss": 0.0147,
+      "num_tokens": 5566712.0,
+      "reward": 0.776611328125,
+      "reward_std": 0.005144121125340462,
+      "rewards//mean": 0.776611328125,
+      "rewards//std": 0.02671085111796856,
+      "step": 644
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.129,
+      "grad_norm": 1.114711880683899,
+      "kl": 0.28322512097656727,
+      "learning_rate": 4.824441214720629e-06,
+      "loss": 0.0113,
+      "num_tokens": 5575304.0,
+      "reward": 0.7742919921875,
+      "reward_std": 0.011971713975071907,
+      "rewards//mean": 0.7742919921875,
+      "rewards//std": 0.034474264830350876,
+      "step": 645
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1292,
+      "grad_norm": 1.0570855140686035,
+      "kl": 0.3079027570784092,
+      "learning_rate": 4.823856657026448e-06,
+      "loss": 0.0123,
+      "num_tokens": 5584080.0,
+      "reward": 0.75958251953125,
+      "reward_std": 0.00948810763657093,
+      "rewards//mean": 0.75958251953125,
+      "rewards//std": 0.031992435455322266,
+      "step": 646
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1294,
+      "grad_norm": 1.053823471069336,
+      "kl": 0.33063437044620514,
+      "learning_rate": 4.823271163283084e-06,
+      "loss": 0.0132,
+      "num_tokens": 5592720.0,
+      "reward": 0.7550048828125,
+      "reward_std": 0.006211505271494389,
+      "rewards//mean": 0.7550048828125,
+      "rewards//std": 0.02459198608994484,
+      "step": 647
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1296,
+      "grad_norm": 0.8944997787475586,
+      "kl": 0.2919867653399706,
+      "learning_rate": 4.822684733726373e-06,
+      "loss": 0.0117,
+      "num_tokens": 5601432.0,
+      "reward": 0.76715087890625,
+      "reward_std": 0.00828872062265873,
+      "rewards//mean": 0.76715087890625,
+      "rewards//std": 0.014392371289432049,
+      "step": 648
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1298,
+      "grad_norm": 1.134657382965088,
+      "kl": 0.2569319698959589,
+      "learning_rate": 4.822097368592529e-06,
+      "loss": 0.0103,
+      "num_tokens": 5610040.0,
+      "reward": 0.723388671875,
+      "reward_std": 0.009949292987585068,
+      "rewards//mean": 0.723388671875,
+      "rewards//std": 0.02867857925593853,
+      "step": 649
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.13,
+      "grad_norm": 0.8463056087493896,
+      "kl": 0.29777566343545914,
+      "learning_rate": 4.821509068118143e-06,
+      "loss": 0.0119,
+      "num_tokens": 5618752.0,
+      "reward": 0.73681640625,
+      "reward_std": 0.00745723582804203,
+      "rewards//mean": 0.73681640625,
+      "rewards//std": 0.026231637224555016,
+      "step": 650
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1302,
+      "grad_norm": 0.9844704270362854,
+      "kl": 0.3054410591721535,
+      "learning_rate": 4.8209198325401815e-06,
+      "loss": 0.0122,
+      "num_tokens": 5627320.0,
+      "reward": 0.76171875,
+      "reward_std": 0.007473442703485489,
+      "rewards//mean": 0.76171875,
+      "rewards//std": 0.01674727164208889,
+      "step": 651
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1304,
+      "grad_norm": 1.0094974040985107,
+      "kl": 0.29557618498802185,
+      "learning_rate": 4.82032966209599e-06,
+      "loss": 0.0118,
+      "num_tokens": 5635952.0,
+      "reward": 0.77386474609375,
+      "reward_std": 0.008985331282019615,
+      "rewards//mean": 0.77386474609375,
+      "rewards//std": 0.028392048552632332,
+      "step": 652
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1306,
+      "grad_norm": 0.8305743336677551,
+      "kl": 0.2749790381640196,
+      "learning_rate": 4.819738557023287e-06,
+      "loss": 0.011,
+      "num_tokens": 5644616.0,
+      "reward": 0.75286865234375,
+      "reward_std": 0.009393520653247833,
+      "rewards//mean": 0.75286865234375,
+      "rewards//std": 0.024232570081949234,
+      "step": 653
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1308,
+      "grad_norm": 0.9685484170913696,
+      "kl": 0.3104813024401665,
+      "learning_rate": 4.819146517560171e-06,
+      "loss": 0.0124,
+      "num_tokens": 5653272.0,
+      "reward": 0.77801513671875,
+      "reward_std": 0.006662518717348576,
+      "rewards//mean": 0.77801513671875,
+      "rewards//std": 0.019912630319595337,
+      "step": 654
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.131,
+      "grad_norm": 1.2605221271514893,
+      "kl": 0.28511145897209644,
+      "learning_rate": 4.818553543945115e-06,
+      "loss": 0.0114,
+      "num_tokens": 5661888.0,
+      "reward": 0.78240966796875,
+      "reward_std": 0.007852917537093163,
+      "rewards//mean": 0.78240966796875,
+      "rewards//std": 0.01976766251027584,
+      "step": 655
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1312,
+      "grad_norm": 0.945095956325531,
+      "kl": 0.2604890577495098,
+      "learning_rate": 4.817959636416969e-06,
+      "loss": 0.0104,
+      "num_tokens": 5670504.0,
+      "reward": 0.73822021484375,
+      "reward_std": 0.00944865308701992,
+      "rewards//mean": 0.73822021484375,
+      "rewards//std": 0.03974169120192528,
+      "step": 656
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1314,
+      "grad_norm": 0.7084172368049622,
+      "kl": 0.24680477194488049,
+      "learning_rate": 4.8173647952149584e-06,
+      "loss": 0.0099,
+      "num_tokens": 5679168.0,
+      "reward": 0.76702880859375,
+      "reward_std": 0.007409450598061085,
+      "rewards//mean": 0.76702880859375,
+      "rewards//std": 0.026313383132219315,
+      "step": 657
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1316,
+      "grad_norm": 0.7127532958984375,
+      "kl": 0.2393797803670168,
+      "learning_rate": 4.816769020578685e-06,
+      "loss": 0.0096,
+      "num_tokens": 5687896.0,
+      "reward": 0.76580810546875,
+      "reward_std": 0.009323619306087494,
+      "rewards//mean": 0.76580810546875,
+      "rewards//std": 0.025117818266153336,
+      "step": 658
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1318,
+      "grad_norm": 0.7750257253646851,
+      "kl": 0.26598621532320976,
+      "learning_rate": 4.816172312748128e-06,
+      "loss": 0.0106,
+      "num_tokens": 5696520.0,
+      "reward": 0.7744140625,
+      "reward_std": 0.008821007795631886,
+      "rewards//mean": 0.7744140625,
+      "rewards//std": 0.015120835043489933,
+      "step": 659
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.132,
+      "grad_norm": 0.6631627082824707,
+      "kl": 0.2553862910717726,
+      "learning_rate": 4.81557467196364e-06,
+      "loss": 0.0102,
+      "num_tokens": 5705160.0,
+      "reward": 0.7291259765625,
+      "reward_std": 0.006548763252794743,
+      "rewards//mean": 0.7291259765625,
+      "rewards//std": 0.022490572184324265,
+      "step": 660
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1322,
+      "grad_norm": 0.5979108810424805,
+      "kl": 0.2634982131421566,
+      "learning_rate": 4.814976098465951e-06,
+      "loss": 0.0105,
+      "num_tokens": 5713848.0,
+      "reward": 0.7772216796875,
+      "reward_std": 0.006953542120754719,
+      "rewards//mean": 0.7772216796875,
+      "rewards//std": 0.017441095784306526,
+      "step": 661
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1324,
+      "grad_norm": 0.6861351728439331,
+      "kl": 0.2535683810710907,
+      "learning_rate": 4.814376592496167e-06,
+      "loss": 0.0101,
+      "num_tokens": 5722440.0,
+      "reward": 0.7781982421875,
+      "reward_std": 0.006485221907496452,
+      "rewards//mean": 0.7781982421875,
+      "rewards//std": 0.028885535895824432,
+      "step": 662
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1326,
+      "grad_norm": 0.6136988997459412,
+      "kl": 0.23714140430092812,
+      "learning_rate": 4.813776154295767e-06,
+      "loss": 0.0095,
+      "num_tokens": 5731056.0,
+      "reward": 0.77496337890625,
+      "reward_std": 0.007635117508471012,
+      "rewards//mean": 0.77496337890625,
+      "rewards//std": 0.0270240418612957,
+      "step": 663
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1328,
+      "grad_norm": 0.7283768653869629,
+      "kl": 0.23214296996593475,
+      "learning_rate": 4.81317478410661e-06,
+      "loss": 0.0093,
+      "num_tokens": 5739704.0,
+      "reward": 0.74163818359375,
+      "reward_std": 0.009544800035655499,
+      "rewards//mean": 0.74163818359375,
+      "rewards//std": 0.030503764748573303,
+      "step": 664
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.133,
+      "grad_norm": 1.0039687156677246,
+      "kl": 0.2293421532958746,
+      "learning_rate": 4.812572482170926e-06,
+      "loss": 0.0092,
+      "num_tokens": 5748272.0,
+      "reward": 0.76214599609375,
+      "reward_std": 0.008817870169878006,
+      "rewards//mean": 0.76214599609375,
+      "rewards//std": 0.02790156379342079,
+      "step": 665
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1332,
+      "grad_norm": 0.7138220071792603,
+      "kl": 0.22582152672111988,
+      "learning_rate": 4.811969248731323e-06,
+      "loss": 0.009,
+      "num_tokens": 5756944.0,
+      "reward": 0.774169921875,
+      "reward_std": 0.009145371615886688,
+      "rewards//mean": 0.774169921875,
+      "rewards//std": 0.024546701461076736,
+      "step": 666
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1334,
+      "grad_norm": 0.6133260726928711,
+      "kl": 0.24057517014443874,
+      "learning_rate": 4.811365084030784e-06,
+      "loss": 0.0096,
+      "num_tokens": 5765464.0,
+      "reward": 0.7408447265625,
+      "reward_std": 0.009205160662531853,
+      "rewards//mean": 0.7408447265625,
+      "rewards//std": 0.03023541159927845,
+      "step": 667
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1336,
+      "grad_norm": 0.6326518058776855,
+      "kl": 0.23078422248363495,
+      "learning_rate": 4.8107599883126634e-06,
+      "loss": 0.0092,
+      "num_tokens": 5774152.0,
+      "reward": 0.76348876953125,
+      "reward_std": 0.010718736797571182,
+      "rewards//mean": 0.76348876953125,
+      "rewards//std": 0.031055085361003876,
+      "step": 668
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1338,
+      "grad_norm": 0.7892494201660156,
+      "kl": 0.2298267874866724,
+      "learning_rate": 4.810153961820697e-06,
+      "loss": 0.0092,
+      "num_tokens": 5782856.0,
+      "reward": 0.76007080078125,
+      "reward_std": 0.00991053506731987,
+      "rewards//mean": 0.76007080078125,
+      "rewards//std": 0.030913403257727623,
+      "step": 669
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.134,
+      "grad_norm": 0.6180135011672974,
+      "kl": 0.21059091202914715,
+      "learning_rate": 4.809547004798991e-06,
+      "loss": 0.0084,
+      "num_tokens": 5791512.0,
+      "reward": 0.7464599609375,
+      "reward_std": 0.011077282950282097,
+      "rewards//mean": 0.7464599609375,
+      "rewards//std": 0.03385930880904198,
+      "step": 670
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1342,
+      "grad_norm": 0.7543632388114929,
+      "kl": 0.2657131403684616,
+      "learning_rate": 4.808939117492028e-06,
+      "loss": 0.0106,
+      "num_tokens": 5800024.0,
+      "reward": 0.73602294921875,
+      "reward_std": 0.009333530440926552,
+      "rewards//mean": 0.73602294921875,
+      "rewards//std": 0.026308204978704453,
+      "step": 671
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1344,
+      "grad_norm": 0.7919978499412537,
+      "kl": 0.2661147303879261,
+      "learning_rate": 4.808330300144664e-06,
+      "loss": 0.0106,
+      "num_tokens": 5808632.0,
+      "reward": 0.75872802734375,
+      "reward_std": 0.009824881330132484,
+      "rewards//mean": 0.75872802734375,
+      "rewards//std": 0.02316456288099289,
+      "step": 672
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1346,
+      "grad_norm": 0.7620828747749329,
+      "kl": 0.25611876510083675,
+      "learning_rate": 4.807720553002132e-06,
+      "loss": 0.0102,
+      "num_tokens": 5817208.0,
+      "reward": 0.7601318359375,
+      "reward_std": 0.009487172588706017,
+      "rewards//mean": 0.7601318359375,
+      "rewards//std": 0.029021471738815308,
+      "step": 673
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1348,
+      "grad_norm": 0.9600464105606079,
+      "kl": 0.24580915831029415,
+      "learning_rate": 4.807109876310037e-06,
+      "loss": 0.0098,
+      "num_tokens": 5826000.0,
+      "reward": 0.7584228515625,
+      "reward_std": 0.008735930547118187,
+      "rewards//mean": 0.7584228515625,
+      "rewards//std": 0.027020471170544624,
+      "step": 674
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.135,
+      "grad_norm": 0.7932077050209045,
+      "kl": 0.251651618629694,
+      "learning_rate": 4.806498270314359e-06,
+      "loss": 0.0101,
+      "num_tokens": 5834584.0,
+      "reward": 0.76025390625,
+      "reward_std": 0.007064810022711754,
+      "rewards//mean": 0.76025390625,
+      "rewards//std": 0.02699608914554119,
+      "step": 675
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1352,
+      "grad_norm": 0.8365455269813538,
+      "kl": 0.2700675167143345,
+      "learning_rate": 4.805885735261454e-06,
+      "loss": 0.0108,
+      "num_tokens": 5843224.0,
+      "reward": 0.7371826171875,
+      "reward_std": 0.010191906243562698,
+      "rewards//mean": 0.7371826171875,
+      "rewards//std": 0.023923013359308243,
+      "step": 676
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1354,
+      "grad_norm": 0.7729324698448181,
+      "kl": 0.26824005506932735,
+      "learning_rate": 4.805272271398051e-06,
+      "loss": 0.0107,
+      "num_tokens": 5851800.0,
+      "reward": 0.70831298828125,
+      "reward_std": 0.008622756227850914,
+      "rewards//mean": 0.70831298828125,
+      "rewards//std": 0.02663075178861618,
+      "step": 677
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1356,
+      "grad_norm": 0.9402274489402771,
+      "kl": 0.2813422940671444,
+      "learning_rate": 4.804657878971252e-06,
+      "loss": 0.0113,
+      "num_tokens": 5860408.0,
+      "reward": 0.74505615234375,
+      "reward_std": 0.007954198867082596,
+      "rewards//mean": 0.74505615234375,
+      "rewards//std": 0.023315679281949997,
+      "step": 678
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1358,
+      "grad_norm": 0.7352375984191895,
+      "kl": 0.26763404347002506,
+      "learning_rate": 4.804042558228535e-06,
+      "loss": 0.0107,
+      "num_tokens": 5869048.0,
+      "reward": 0.7579345703125,
+      "reward_std": 0.006437701638787985,
+      "rewards//mean": 0.7579345703125,
+      "rewards//std": 0.029943620786070824,
+      "step": 679
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.136,
+      "grad_norm": 0.8864990472793579,
+      "kl": 0.3166388310492039,
+      "learning_rate": 4.803426309417752e-06,
+      "loss": 0.0127,
+      "num_tokens": 5877704.0,
+      "reward": 0.75897216796875,
+      "reward_std": 0.007796045392751694,
+      "rewards//mean": 0.75897216796875,
+      "rewards//std": 0.0265179630368948,
+      "step": 680
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1362,
+      "grad_norm": 0.8726670742034912,
+      "kl": 0.28164214082062244,
+      "learning_rate": 4.802809132787125e-06,
+      "loss": 0.0113,
+      "num_tokens": 5886232.0,
+      "reward": 0.74322509765625,
+      "reward_std": 0.007162688300013542,
+      "rewards//mean": 0.74322509765625,
+      "rewards//std": 0.024352213367819786,
+      "step": 681
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1364,
+      "grad_norm": 1.2566941976547241,
+      "kl": 0.3058667443692684,
+      "learning_rate": 4.802191028585257e-06,
+      "loss": 0.0122,
+      "num_tokens": 5894872.0,
+      "reward": 0.736328125,
+      "reward_std": 0.009593142196536064,
+      "rewards//mean": 0.736328125,
+      "rewards//std": 0.04730703681707382,
+      "step": 682
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1366,
+      "grad_norm": 0.901271641254425,
+      "kl": 0.3303819112479687,
+      "learning_rate": 4.801571997061117e-06,
+      "loss": 0.0132,
+      "num_tokens": 5903480.0,
+      "reward": 0.7298583984375,
+      "reward_std": 0.008278006687760353,
+      "rewards//mean": 0.7298583984375,
+      "rewards//std": 0.03159085661172867,
+      "step": 683
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1368,
+      "grad_norm": 0.846440851688385,
+      "kl": 0.2713386472314596,
+      "learning_rate": 4.800952038464051e-06,
+      "loss": 0.0109,
+      "num_tokens": 5912088.0,
+      "reward": 0.753173828125,
+      "reward_std": 0.008396659046411514,
+      "rewards//mean": 0.753173828125,
+      "rewards//std": 0.02986997365951538,
+      "step": 684
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.137,
+      "grad_norm": 0.99683678150177,
+      "kl": 0.29254608042538166,
+      "learning_rate": 4.800331153043781e-06,
+      "loss": 0.0117,
+      "num_tokens": 5920760.0,
+      "reward": 0.7811279296875,
+      "reward_std": 0.006906645372509956,
+      "rewards//mean": 0.7811279296875,
+      "rewards//std": 0.02221698872745037,
+      "step": 685
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1372,
+      "grad_norm": 0.9804807305335999,
+      "kl": 0.32390232756733894,
+      "learning_rate": 4.799709341050397e-06,
+      "loss": 0.013,
+      "num_tokens": 5929320.0,
+      "reward": 0.75372314453125,
+      "reward_std": 0.009741474874317646,
+      "rewards//mean": 0.75372314453125,
+      "rewards//std": 0.022919517010450363,
+      "step": 686
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1374,
+      "grad_norm": 1.048980712890625,
+      "kl": 0.3197479471564293,
+      "learning_rate": 4.799086602734364e-06,
+      "loss": 0.0128,
+      "num_tokens": 5937912.0,
+      "reward": 0.732177734375,
+      "reward_std": 0.011059543117880821,
+      "rewards//mean": 0.732177734375,
+      "rewards//std": 0.03457533195614815,
+      "step": 687
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1376,
+      "grad_norm": 0.9630193114280701,
+      "kl": 0.33098217844963074,
+      "learning_rate": 4.798462938346524e-06,
+      "loss": 0.0132,
+      "num_tokens": 5946624.0,
+      "reward": 0.74591064453125,
+      "reward_std": 0.008718679659068584,
+      "rewards//mean": 0.74591064453125,
+      "rewards//std": 0.0328625813126564,
+      "step": 688
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1378,
+      "grad_norm": 1.1587707996368408,
+      "kl": 0.29454705864191055,
+      "learning_rate": 4.7978383481380865e-06,
+      "loss": 0.0118,
+      "num_tokens": 5955192.0,
+      "reward": 0.7099609375,
+      "reward_std": 0.007809010334312916,
+      "rewards//mean": 0.7099609375,
+      "rewards//std": 0.034053899347782135,
+      "step": 689
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.138,
+      "grad_norm": 0.9946979284286499,
+      "kl": 0.32380471006035805,
+      "learning_rate": 4.797212832360637e-06,
+      "loss": 0.013,
+      "num_tokens": 5963760.0,
+      "reward": 0.767822265625,
+      "reward_std": 0.00826583243906498,
+      "rewards//mean": 0.767822265625,
+      "rewards//std": 0.02232472226023674,
+      "step": 690
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1382,
+      "grad_norm": 1.1503069400787354,
+      "kl": 0.3658827282488346,
+      "learning_rate": 4.796586391266135e-06,
+      "loss": 0.0146,
+      "num_tokens": 5972352.0,
+      "reward": 0.750732421875,
+      "reward_std": 0.0072412192821502686,
+      "rewards//mean": 0.750732421875,
+      "rewards//std": 0.031179197132587433,
+      "step": 691
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1384,
+      "grad_norm": 1.0804966688156128,
+      "kl": 0.3435588367283344,
+      "learning_rate": 4.795959025106907e-06,
+      "loss": 0.0137,
+      "num_tokens": 5980920.0,
+      "reward": 0.75189208984375,
+      "reward_std": 0.009941812604665756,
+      "rewards//mean": 0.75189208984375,
+      "rewards//std": 0.02548695169389248,
+      "step": 692
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1386,
+      "grad_norm": 1.247780442237854,
+      "kl": 0.3512617312371731,
+      "learning_rate": 4.7953307341356595e-06,
+      "loss": 0.0141,
+      "num_tokens": 5989576.0,
+      "reward": 0.765869140625,
+      "reward_std": 0.007969655096530914,
+      "rewards//mean": 0.765869140625,
+      "rewards//std": 0.028762908652424812,
+      "step": 693
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1388,
+      "grad_norm": 1.2001007795333862,
+      "kl": 0.36641981452703476,
+      "learning_rate": 4.794701518605467e-06,
+      "loss": 0.0147,
+      "num_tokens": 5998160.0,
+      "reward": 0.763427734375,
+      "reward_std": 0.0078741405159235,
+      "rewards//mean": 0.763427734375,
+      "rewards//std": 0.024566426873207092,
+      "step": 694
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.139,
+      "grad_norm": 1.1044167280197144,
+      "kl": 0.33926333859562874,
+      "learning_rate": 4.794071378769776e-06,
+      "loss": 0.0136,
+      "num_tokens": 6006776.0,
+      "reward": 0.76068115234375,
+      "reward_std": 0.01240166462957859,
+      "rewards//mean": 0.76068115234375,
+      "rewards//std": 0.03412666544318199,
+      "step": 695
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1392,
+      "grad_norm": 1.0884336233139038,
+      "kl": 0.3415757305920124,
+      "learning_rate": 4.7934403148824085e-06,
+      "loss": 0.0137,
+      "num_tokens": 6015376.0,
+      "reward": 0.7767333984375,
+      "reward_std": 0.00934339314699173,
+      "rewards//mean": 0.7767333984375,
+      "rewards//std": 0.023022731766104698,
+      "step": 696
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1394,
+      "grad_norm": 1.1337672472000122,
+      "kl": 0.3384525291621685,
+      "learning_rate": 4.792808327197556e-06,
+      "loss": 0.0135,
+      "num_tokens": 6023976.0,
+      "reward": 0.75604248046875,
+      "reward_std": 0.011509222909808159,
+      "rewards//mean": 0.75604248046875,
+      "rewards//std": 0.03885067626833916,
+      "step": 697
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1396,
+      "grad_norm": 1.2506954669952393,
+      "kl": 0.3618217594921589,
+      "learning_rate": 4.792175415969786e-06,
+      "loss": 0.0145,
+      "num_tokens": 6032584.0,
+      "reward": 0.78179931640625,
+      "reward_std": 0.007064157165586948,
+      "rewards//mean": 0.78179931640625,
+      "rewards//std": 0.023384397849440575,
+      "step": 698
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1398,
+      "grad_norm": 1.190083384513855,
+      "kl": 0.3704114742577076,
+      "learning_rate": 4.79154158145403e-06,
+      "loss": 0.0148,
+      "num_tokens": 6041200.0,
+      "reward": 0.7354736328125,
+      "reward_std": 0.007111767306923866,
+      "rewards//mean": 0.7354736328125,
+      "rewards//std": 0.03445318341255188,
+      "step": 699
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.14,
+      "grad_norm": 1.0952671766281128,
+      "kl": 0.3880801610648632,
+      "learning_rate": 4.790906823905599e-06,
+      "loss": 0.0155,
+      "num_tokens": 6049776.0,
+      "reward": 0.75994873046875,
+      "reward_std": 0.006981437094509602,
+      "rewards//mean": 0.75994873046875,
+      "rewards//std": 0.02911488339304924,
+      "step": 700
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1402,
+      "grad_norm": 0.9023262858390808,
+      "kl": 0.3351786471903324,
+      "learning_rate": 4.790271143580174e-06,
+      "loss": 0.0134,
+      "num_tokens": 6058352.0,
+      "reward": 0.76885986328125,
+      "reward_std": 0.008710931986570358,
+      "rewards//mean": 0.76885986328125,
+      "rewards//std": 0.03886703774333,
+      "step": 701
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1404,
+      "grad_norm": 1.2910362482070923,
+      "kl": 0.3798423409461975,
+      "learning_rate": 4.789634540733807e-06,
+      "loss": 0.0152,
+      "num_tokens": 6066912.0,
+      "reward": 0.748046875,
+      "reward_std": 0.006378927733749151,
+      "rewards//mean": 0.748046875,
+      "rewards//std": 0.029968129470944405,
+      "step": 702
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1406,
+      "grad_norm": 1.287479043006897,
+      "kl": 0.37538506276905537,
+      "learning_rate": 4.78899701562292e-06,
+      "loss": 0.015,
+      "num_tokens": 6075536.0,
+      "reward": 0.75531005859375,
+      "reward_std": 0.010323834605515003,
+      "rewards//mean": 0.75531005859375,
+      "rewards//std": 0.029662420973181725,
+      "step": 703
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1408,
+      "grad_norm": 1.0325592756271362,
+      "kl": 0.35265588015317917,
+      "learning_rate": 4.788358568504308e-06,
+      "loss": 0.0141,
+      "num_tokens": 6084152.0,
+      "reward": 0.73480224609375,
+      "reward_std": 0.007402766961604357,
+      "rewards//mean": 0.73480224609375,
+      "rewards//std": 0.034619029611349106,
+      "step": 704
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.141,
+      "grad_norm": 1.6814100742340088,
+      "kl": 0.38387027755379677,
+      "learning_rate": 4.78771919963514e-06,
+      "loss": 0.0154,
+      "num_tokens": 6092728.0,
+      "reward": 0.79052734375,
+      "reward_std": 0.006797960493713617,
+      "rewards//mean": 0.79052734375,
+      "rewards//std": 0.020333237946033478,
+      "step": 705
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1412,
+      "grad_norm": 1.1181515455245972,
+      "kl": 0.3311033882200718,
+      "learning_rate": 4.787078909272951e-06,
+      "loss": 0.0132,
+      "num_tokens": 6101480.0,
+      "reward": 0.73321533203125,
+      "reward_std": 0.012039251625537872,
+      "rewards//mean": 0.73321533203125,
+      "rewards//std": 0.04149588197469711,
+      "step": 706
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1414,
+      "grad_norm": 1.1418641805648804,
+      "kl": 0.2598997447639704,
+      "learning_rate": 4.786437697675651e-06,
+      "loss": 0.0104,
+      "num_tokens": 6110208.0,
+      "reward": 0.73748779296875,
+      "reward_std": 0.01649647206068039,
+      "rewards//mean": 0.73748779296875,
+      "rewards//std": 0.04361593350768089,
+      "step": 707
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1416,
+      "grad_norm": 1.1949273347854614,
+      "kl": 0.39064906910061836,
+      "learning_rate": 4.78579556510152e-06,
+      "loss": 0.0156,
+      "num_tokens": 6118832.0,
+      "reward": 0.74700927734375,
+      "reward_std": 0.009526428766548634,
+      "rewards//mean": 0.74700927734375,
+      "rewards//std": 0.02997412718832493,
+      "step": 708
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1418,
+      "grad_norm": 1.232944369316101,
+      "kl": 0.3855067417025566,
+      "learning_rate": 4.785152511809208e-06,
+      "loss": 0.0154,
+      "num_tokens": 6127480.0,
+      "reward": 0.751220703125,
+      "reward_std": 0.0074347625486552715,
+      "rewards//mean": 0.751220703125,
+      "rewards//std": 0.0333704799413681,
+      "step": 709
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.142,
+      "grad_norm": 0.8768677711486816,
+      "kl": 0.3823428861796856,
+      "learning_rate": 4.784508538057738e-06,
+      "loss": 0.0153,
+      "num_tokens": 6136016.0,
+      "reward": 0.734375,
+      "reward_std": 0.009139763191342354,
+      "rewards//mean": 0.734375,
+      "rewards//std": 0.029879095032811165,
+      "step": 710
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1422,
+      "grad_norm": 1.023090124130249,
+      "kl": 0.3618377633392811,
+      "learning_rate": 4.783863644106502e-06,
+      "loss": 0.0145,
+      "num_tokens": 6144688.0,
+      "reward": 0.75811767578125,
+      "reward_std": 0.007518475875258446,
+      "rewards//mean": 0.75811767578125,
+      "rewards//std": 0.02010328881442547,
+      "step": 711
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1424,
+      "grad_norm": 0.9730324149131775,
+      "kl": 0.36088190227746964,
+      "learning_rate": 4.783217830215264e-06,
+      "loss": 0.0144,
+      "num_tokens": 6153288.0,
+      "reward": 0.7520751953125,
+      "reward_std": 0.0069280690513551235,
+      "rewards//mean": 0.7520751953125,
+      "rewards//std": 0.03221152722835541,
+      "step": 712
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1426,
+      "grad_norm": 0.777560830116272,
+      "kl": 0.3026167619973421,
+      "learning_rate": 4.782571096644157e-06,
+      "loss": 0.0121,
+      "num_tokens": 6161936.0,
+      "reward": 0.75244140625,
+      "reward_std": 0.012461979873478413,
+      "rewards//mean": 0.75244140625,
+      "rewards//std": 0.04013592377305031,
+      "step": 713
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1428,
+      "grad_norm": 0.7599678635597229,
+      "kl": 0.27103759720921516,
+      "learning_rate": 4.7819234436536845e-06,
+      "loss": 0.0108,
+      "num_tokens": 6170696.0,
+      "reward": 0.77105712890625,
+      "reward_std": 0.009724288247525692,
+      "rewards//mean": 0.77105712890625,
+      "rewards//std": 0.028811221942305565,
+      "step": 714
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.143,
+      "grad_norm": 1.2590875625610352,
+      "kl": 0.3248850591480732,
+      "learning_rate": 4.781274871504722e-06,
+      "loss": 0.013,
+      "num_tokens": 6179384.0,
+      "reward": 0.7764892578125,
+      "reward_std": 0.005956442095339298,
+      "rewards//mean": 0.7764892578125,
+      "rewards//std": 0.02729027532041073,
+      "step": 715
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1432,
+      "grad_norm": 0.8374090194702148,
+      "kl": 0.3235581796616316,
+      "learning_rate": 4.780625380458513e-06,
+      "loss": 0.0129,
+      "num_tokens": 6187896.0,
+      "reward": 0.75469970703125,
+      "reward_std": 0.008836697787046432,
+      "rewards//mean": 0.75469970703125,
+      "rewards//std": 0.03515404462814331,
+      "step": 716
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1434,
+      "grad_norm": 0.9022362232208252,
+      "kl": 0.3358203247189522,
+      "learning_rate": 4.7799749707766754e-06,
+      "loss": 0.0134,
+      "num_tokens": 6196536.0,
+      "reward": 0.73065185546875,
+      "reward_std": 0.006113333627581596,
+      "rewards//mean": 0.73065185546875,
+      "rewards//std": 0.025127459317445755,
+      "step": 717
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1436,
+      "grad_norm": 1.1146235466003418,
+      "kl": 0.3147228341549635,
+      "learning_rate": 4.779323642721191e-06,
+      "loss": 0.0126,
+      "num_tokens": 6205168.0,
+      "reward": 0.7520751953125,
+      "reward_std": 0.007163808681070805,
+      "rewards//mean": 0.7520751953125,
+      "rewards//std": 0.019917665049433708,
+      "step": 718
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1438,
+      "grad_norm": 0.9956210851669312,
+      "kl": 0.3257326614111662,
+      "learning_rate": 4.778671396554417e-06,
+      "loss": 0.013,
+      "num_tokens": 6213784.0,
+      "reward": 0.76202392578125,
+      "reward_std": 0.009784232825040817,
+      "rewards//mean": 0.76202392578125,
+      "rewards//std": 0.027377430349588394,
+      "step": 719
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.144,
+      "grad_norm": 1.103261947631836,
+      "kl": 0.31043668277561665,
+      "learning_rate": 4.778018232539075e-06,
+      "loss": 0.0124,
+      "num_tokens": 6222432.0,
+      "reward": 0.75640869140625,
+      "reward_std": 0.010043056681752205,
+      "rewards//mean": 0.75640869140625,
+      "rewards//std": 0.026082253083586693,
+      "step": 720
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1442,
+      "grad_norm": 1.2115051746368408,
+      "kl": 0.30454549565911293,
+      "learning_rate": 4.777364150938263e-06,
+      "loss": 0.0122,
+      "num_tokens": 6231232.0,
+      "reward": 0.755859375,
+      "reward_std": 0.01059969887137413,
+      "rewards//mean": 0.755859375,
+      "rewards//std": 0.034273672848939896,
+      "step": 721
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1444,
+      "grad_norm": 0.8916050791740417,
+      "kl": 0.2920965179800987,
+      "learning_rate": 4.776709152015443e-06,
+      "loss": 0.0117,
+      "num_tokens": 6239984.0,
+      "reward": 0.74688720703125,
+      "reward_std": 0.007067927625030279,
+      "rewards//mean": 0.74688720703125,
+      "rewards//std": 0.025057479739189148,
+      "step": 722
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1446,
+      "grad_norm": 0.7612753510475159,
+      "kl": 0.29840802773833275,
+      "learning_rate": 4.776053236034449e-06,
+      "loss": 0.0119,
+      "num_tokens": 6248576.0,
+      "reward": 0.7445068359375,
+      "reward_std": 0.00902190525084734,
+      "rewards//mean": 0.7445068359375,
+      "rewards//std": 0.02381392940878868,
+      "step": 723
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1448,
+      "grad_norm": 1.3189657926559448,
+      "kl": 0.3207559399306774,
+      "learning_rate": 4.775396403259483e-06,
+      "loss": 0.0128,
+      "num_tokens": 6257128.0,
+      "reward": 0.7669677734375,
+      "reward_std": 0.01015978679060936,
+      "rewards//mean": 0.7669677734375,
+      "rewards//std": 0.026062294840812683,
+      "step": 724
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.145,
+      "grad_norm": 0.9157844185829163,
+      "kl": 0.30339373275637627,
+      "learning_rate": 4.774738653955119e-06,
+      "loss": 0.0121,
+      "num_tokens": 6265768.0,
+      "reward": 0.7802734375,
+      "reward_std": 0.009250717237591743,
+      "rewards//mean": 0.7802734375,
+      "rewards//std": 0.026991603896021843,
+      "step": 725
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1452,
+      "grad_norm": 1.1949882507324219,
+      "kl": 0.33243946731090546,
+      "learning_rate": 4.7740799883862966e-06,
+      "loss": 0.0133,
+      "num_tokens": 6274448.0,
+      "reward": 0.7586669921875,
+      "reward_std": 0.01018530037254095,
+      "rewards//mean": 0.7586669921875,
+      "rewards//std": 0.020966242998838425,
+      "step": 726
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1454,
+      "grad_norm": 0.8508477807044983,
+      "kl": 0.33175548166036606,
+      "learning_rate": 4.773420406818327e-06,
+      "loss": 0.0133,
+      "num_tokens": 6283040.0,
+      "reward": 0.75091552734375,
+      "reward_std": 0.007159090135246515,
+      "rewards//mean": 0.75091552734375,
+      "rewards//std": 0.02595542185008526,
+      "step": 727
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1456,
+      "grad_norm": 0.8403224349021912,
+      "kl": 0.30662048049271107,
+      "learning_rate": 4.772759909516889e-06,
+      "loss": 0.0123,
+      "num_tokens": 6291704.0,
+      "reward": 0.7113037109375,
+      "reward_std": 0.011537307873368263,
+      "rewards//mean": 0.7113037109375,
+      "rewards//std": 0.0381256639957428,
+      "step": 728
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1458,
+      "grad_norm": 1.1425209045410156,
+      "kl": 0.31275169365108013,
+      "learning_rate": 4.772098496748031e-06,
+      "loss": 0.0125,
+      "num_tokens": 6300344.0,
+      "reward": 0.75421142578125,
+      "reward_std": 0.00868441816419363,
+      "rewards//mean": 0.75421142578125,
+      "rewards//std": 0.03845828026533127,
+      "step": 729
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.146,
+      "grad_norm": 0.7371503710746765,
+      "kl": 0.2826213352382183,
+      "learning_rate": 4.7714361687781705e-06,
+      "loss": 0.0113,
+      "num_tokens": 6308912.0,
+      "reward": 0.75616455078125,
+      "reward_std": 0.01087634265422821,
+      "rewards//mean": 0.75616455078125,
+      "rewards//std": 0.02745252661406994,
+      "step": 730
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1462,
+      "grad_norm": 1.2550262212753296,
+      "kl": 0.29363943450152874,
+      "learning_rate": 4.770772925874093e-06,
+      "loss": 0.0117,
+      "num_tokens": 6317680.0,
+      "reward": 0.7362060546875,
+      "reward_std": 0.012717410922050476,
+      "rewards//mean": 0.7362060546875,
+      "rewards//std": 0.031156614422798157,
+      "step": 731
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1464,
+      "grad_norm": 1.0804203748703003,
+      "kl": 0.347552876919508,
+      "learning_rate": 4.770108768302953e-06,
+      "loss": 0.0139,
+      "num_tokens": 6326368.0,
+      "reward": 0.79925537109375,
+      "reward_std": 0.00879729725420475,
+      "rewards//mean": 0.79925537109375,
+      "rewards//std": 0.021716952323913574,
+      "step": 732
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1466,
+      "grad_norm": 0.8175789713859558,
+      "kl": 0.2651336081326008,
+      "learning_rate": 4.769443696332272e-06,
+      "loss": 0.0106,
+      "num_tokens": 6335016.0,
+      "reward": 0.75885009765625,
+      "reward_std": 0.010360082611441612,
+      "rewards//mean": 0.75885009765625,
+      "rewards//std": 0.0343741700053215,
+      "step": 733
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1468,
+      "grad_norm": 0.9594478011131287,
+      "kl": 0.32585857063531876,
+      "learning_rate": 4.768777710229941e-06,
+      "loss": 0.013,
+      "num_tokens": 6343744.0,
+      "reward": 0.76837158203125,
+      "reward_std": 0.006379896309226751,
+      "rewards//mean": 0.76837158203125,
+      "rewards//std": 0.02404632233083248,
+      "step": 734
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.147,
+      "grad_norm": 0.9124557971954346,
+      "kl": 0.3457811623811722,
+      "learning_rate": 4.768110810264221e-06,
+      "loss": 0.0138,
+      "num_tokens": 6352456.0,
+      "reward": 0.77996826171875,
+      "reward_std": 0.010944314301013947,
+      "rewards//mean": 0.77996826171875,
+      "rewards//std": 0.025017576292157173,
+      "step": 735
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1472,
+      "grad_norm": 1.0064573287963867,
+      "kl": 0.32626586966216564,
+      "learning_rate": 4.767442996703737e-06,
+      "loss": 0.0131,
+      "num_tokens": 6361208.0,
+      "reward": 0.76898193359375,
+      "reward_std": 0.009524751454591751,
+      "rewards//mean": 0.76898193359375,
+      "rewards//std": 0.016290800645947456,
+      "step": 736
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1474,
+      "grad_norm": 0.8128183484077454,
+      "kl": 0.3099711798131466,
+      "learning_rate": 4.7667742698174855e-06,
+      "loss": 0.0124,
+      "num_tokens": 6369800.0,
+      "reward": 0.759765625,
+      "reward_std": 0.00793082732707262,
+      "rewards//mean": 0.759765625,
+      "rewards//std": 0.02536318637430668,
+      "step": 737
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1476,
+      "grad_norm": 0.9029561281204224,
+      "kl": 0.34095400758087635,
+      "learning_rate": 4.766104629874829e-06,
+      "loss": 0.0136,
+      "num_tokens": 6378480.0,
+      "reward": 0.7481689453125,
+      "reward_std": 0.008078465238213539,
+      "rewards//mean": 0.7481689453125,
+      "rewards//std": 0.03140438720583916,
+      "step": 738
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1478,
+      "grad_norm": 1.104424238204956,
+      "kl": 0.3173263669013977,
+      "learning_rate": 4.765434077145499e-06,
+      "loss": 0.0127,
+      "num_tokens": 6387056.0,
+      "reward": 0.74407958984375,
+      "reward_std": 0.01128837838768959,
+      "rewards//mean": 0.74407958984375,
+      "rewards//std": 0.04236414283514023,
+      "step": 739
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.148,
+      "grad_norm": 1.0675055980682373,
+      "kl": 0.3194386400282383,
+      "learning_rate": 4.764762611899593e-06,
+      "loss": 0.0128,
+      "num_tokens": 6395720.0,
+      "reward": 0.779296875,
+      "reward_std": 0.00692769093438983,
+      "rewards//mean": 0.779296875,
+      "rewards//std": 0.02041051536798477,
+      "step": 740
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1482,
+      "grad_norm": 1.5919986963272095,
+      "kl": 0.3498740680515766,
+      "learning_rate": 4.764090234407578e-06,
+      "loss": 0.014,
+      "num_tokens": 6404408.0,
+      "reward": 0.7606201171875,
+      "reward_std": 0.007645751349627972,
+      "rewards//mean": 0.7606201171875,
+      "rewards//std": 0.017224503681063652,
+      "step": 741
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1484,
+      "grad_norm": 1.439361810684204,
+      "kl": 0.33540985360741615,
+      "learning_rate": 4.763416944940287e-06,
+      "loss": 0.0134,
+      "num_tokens": 6413088.0,
+      "reward": 0.7662353515625,
+      "reward_std": 0.011973317712545395,
+      "rewards//mean": 0.7662353515625,
+      "rewards//std": 0.02613653428852558,
+      "step": 742
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1486,
+      "grad_norm": 0.9124048352241516,
+      "kl": 0.3327603731304407,
+      "learning_rate": 4.762742743768921e-06,
+      "loss": 0.0133,
+      "num_tokens": 6421912.0,
+      "reward": 0.782470703125,
+      "reward_std": 0.00991036370396614,
+      "rewards//mean": 0.782470703125,
+      "rewards//std": 0.026327257975935936,
+      "step": 743
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1488,
+      "grad_norm": 1.1488804817199707,
+      "kl": 0.3750241808593273,
+      "learning_rate": 4.762067631165049e-06,
+      "loss": 0.015,
+      "num_tokens": 6430552.0,
+      "reward": 0.78814697265625,
+      "reward_std": 0.01083114929497242,
+      "rewards//mean": 0.78814697265625,
+      "rewards//std": 0.023038752377033234,
+      "step": 744
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.149,
+      "grad_norm": 1.0288392305374146,
+      "kl": 0.4040727950632572,
+      "learning_rate": 4.761391607400606e-06,
+      "loss": 0.0162,
+      "num_tokens": 6439144.0,
+      "reward": 0.774658203125,
+      "reward_std": 0.008166758343577385,
+      "rewards//mean": 0.774658203125,
+      "rewards//std": 0.01806734874844551,
+      "step": 745
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1492,
+      "grad_norm": 1.1654855012893677,
+      "kl": 0.3817331902682781,
+      "learning_rate": 4.7607146727478935e-06,
+      "loss": 0.0153,
+      "num_tokens": 6447728.0,
+      "reward": 0.7738037109375,
+      "reward_std": 0.00860125944018364,
+      "rewards//mean": 0.7738037109375,
+      "rewards//std": 0.018528563901782036,
+      "step": 746
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1494,
+      "grad_norm": 1.4023302793502808,
+      "kl": 0.3670612499117851,
+      "learning_rate": 4.760036827479582e-06,
+      "loss": 0.0147,
+      "num_tokens": 6456312.0,
+      "reward": 0.76483154296875,
+      "reward_std": 0.007299549877643585,
+      "rewards//mean": 0.76483154296875,
+      "rewards//std": 0.028815951198339462,
+      "step": 747
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1496,
+      "grad_norm": 0.9892433285713196,
+      "kl": 0.3555135168135166,
+      "learning_rate": 4.759358071868705e-06,
+      "loss": 0.0142,
+      "num_tokens": 6464936.0,
+      "reward": 0.78558349609375,
+      "reward_std": 0.007815414108335972,
+      "rewards//mean": 0.78558349609375,
+      "rewards//std": 0.016691023483872414,
+      "step": 748
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1498,
+      "grad_norm": 0.9962619543075562,
+      "kl": 0.33968275785446167,
+      "learning_rate": 4.758678406188668e-06,
+      "loss": 0.0136,
+      "num_tokens": 6473664.0,
+      "reward": 0.73126220703125,
+      "reward_std": 0.007433678954839706,
+      "rewards//mean": 0.73126220703125,
+      "rewards//std": 0.032445359975099564,
+      "step": 749
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.15,
+      "grad_norm": 1.2323095798492432,
+      "kl": 0.3530998080968857,
+      "learning_rate": 4.757997830713239e-06,
+      "loss": 0.0141,
+      "num_tokens": 6482200.0,
+      "reward": 0.74542236328125,
+      "reward_std": 0.01127916481345892,
+      "rewards//mean": 0.74542236328125,
+      "rewards//std": 0.019168945029377937,
+      "step": 750
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1502,
+      "grad_norm": 1.0545332431793213,
+      "kl": 0.34781191498041153,
+      "learning_rate": 4.757316345716554e-06,
+      "loss": 0.0139,
+      "num_tokens": 6490712.0,
+      "reward": 0.74566650390625,
+      "reward_std": 0.009755873121321201,
+      "rewards//mean": 0.74566650390625,
+      "rewards//std": 0.03269031271338463,
+      "step": 751
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1504,
+      "grad_norm": 0.9189953804016113,
+      "kl": 0.3220175448805094,
+      "learning_rate": 4.756633951473114e-06,
+      "loss": 0.0129,
+      "num_tokens": 6499392.0,
+      "reward": 0.7847900390625,
+      "reward_std": 0.008869939483702183,
+      "rewards//mean": 0.7847900390625,
+      "rewards//std": 0.021482614800333977,
+      "step": 752
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1506,
+      "grad_norm": 1.1692512035369873,
+      "kl": 0.33156827092170715,
+      "learning_rate": 4.755950648257789e-06,
+      "loss": 0.0133,
+      "num_tokens": 6508160.0,
+      "reward": 0.77008056640625,
+      "reward_std": 0.011027710512280464,
+      "rewards//mean": 0.77008056640625,
+      "rewards//std": 0.02765524946153164,
+      "step": 753
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1508,
+      "grad_norm": 1.144055724143982,
+      "kl": 0.3382889721542597,
+      "learning_rate": 4.755266436345812e-06,
+      "loss": 0.0135,
+      "num_tokens": 6516832.0,
+      "reward": 0.72998046875,
+      "reward_std": 0.01025819219648838,
+      "rewards//mean": 0.72998046875,
+      "rewards//std": 0.029343342408537865,
+      "step": 754
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.151,
+      "grad_norm": 1.0410165786743164,
+      "kl": 0.3598978705704212,
+      "learning_rate": 4.754581316012785e-06,
+      "loss": 0.0144,
+      "num_tokens": 6525520.0,
+      "reward": 0.7647705078125,
+      "reward_std": 0.007099618669599295,
+      "rewards//mean": 0.7647705078125,
+      "rewards//std": 0.02260068617761135,
+      "step": 755
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1512,
+      "grad_norm": 0.9105063080787659,
+      "kl": 0.30748361349105835,
+      "learning_rate": 4.753895287534673e-06,
+      "loss": 0.0123,
+      "num_tokens": 6534128.0,
+      "reward": 0.7608642578125,
+      "reward_std": 0.011949615553021431,
+      "rewards//mean": 0.7608642578125,
+      "rewards//std": 0.036653582006692886,
+      "step": 756
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1514,
+      "grad_norm": 0.8369845747947693,
+      "kl": 0.3764992021024227,
+      "learning_rate": 4.753208351187809e-06,
+      "loss": 0.0151,
+      "num_tokens": 6542736.0,
+      "reward": 0.7330322265625,
+      "reward_std": 0.0075618005357682705,
+      "rewards//mean": 0.7330322265625,
+      "rewards//std": 0.03163682669401169,
+      "step": 757
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1516,
+      "grad_norm": 0.9112576842308044,
+      "kl": 0.340505413711071,
+      "learning_rate": 4.75252050724889e-06,
+      "loss": 0.0136,
+      "num_tokens": 6551352.0,
+      "reward": 0.75885009765625,
+      "reward_std": 0.008580232039093971,
+      "rewards//mean": 0.75885009765625,
+      "rewards//std": 0.024510828778147697,
+      "step": 758
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1518,
+      "grad_norm": 1.1520575284957886,
+      "kl": 0.3449602983891964,
+      "learning_rate": 4.751831755994981e-06,
+      "loss": 0.0138,
+      "num_tokens": 6560040.0,
+      "reward": 0.7685546875,
+      "reward_std": 0.0092132817953825,
+      "rewards//mean": 0.7685546875,
+      "rewards//std": 0.02560080774128437,
+      "step": 759
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.152,
+      "grad_norm": 1.0891118049621582,
+      "kl": 0.3134535998106003,
+      "learning_rate": 4.75114209770351e-06,
+      "loss": 0.0125,
+      "num_tokens": 6568704.0,
+      "reward": 0.74053955078125,
+      "reward_std": 0.009147681295871735,
+      "rewards//mean": 0.74053955078125,
+      "rewards//std": 0.028252549469470978,
+      "step": 760
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1522,
+      "grad_norm": 0.9451463222503662,
+      "kl": 0.35255878791213036,
+      "learning_rate": 4.75045153265227e-06,
+      "loss": 0.0141,
+      "num_tokens": 6577336.0,
+      "reward": 0.7484130859375,
+      "reward_std": 0.0063913362100720406,
+      "rewards//mean": 0.7484130859375,
+      "rewards//std": 0.02723030373454094,
+      "step": 761
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1524,
+      "grad_norm": 0.8575259447097778,
+      "kl": 0.3487287014722824,
+      "learning_rate": 4.749760061119423e-06,
+      "loss": 0.0139,
+      "num_tokens": 6585944.0,
+      "reward": 0.75885009765625,
+      "reward_std": 0.007511901669204235,
+      "rewards//mean": 0.75885009765625,
+      "rewards//std": 0.031213102862238884,
+      "step": 762
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1526,
+      "grad_norm": 0.9670186042785645,
+      "kl": 0.31115617975592613,
+      "learning_rate": 4.749067683383491e-06,
+      "loss": 0.0124,
+      "num_tokens": 6594568.0,
+      "reward": 0.75286865234375,
+      "reward_std": 0.009954852983355522,
+      "rewards//mean": 0.75286865234375,
+      "rewards//std": 0.027128031477332115,
+      "step": 763
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1528,
+      "grad_norm": 1.3793588876724243,
+      "kl": 0.3482302203774452,
+      "learning_rate": 4.748374399723366e-06,
+      "loss": 0.0139,
+      "num_tokens": 6603064.0,
+      "reward": 0.74713134765625,
+      "reward_std": 0.011073645204305649,
+      "rewards//mean": 0.74713134765625,
+      "rewards//std": 0.028673233464360237,
+      "step": 764
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.153,
+      "grad_norm": 1.212506651878357,
+      "kl": 0.35477422177791595,
+      "learning_rate": 4.747680210418302e-06,
+      "loss": 0.0142,
+      "num_tokens": 6611760.0,
+      "reward": 0.76275634765625,
+      "reward_std": 0.008897896856069565,
+      "rewards//mean": 0.76275634765625,
+      "rewards//std": 0.024282492697238922,
+      "step": 765
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1532,
+      "grad_norm": 1.8687018156051636,
+      "kl": 0.33726025745272636,
+      "learning_rate": 4.746985115747918e-06,
+      "loss": 0.0135,
+      "num_tokens": 6620544.0,
+      "reward": 0.788818359375,
+      "reward_std": 0.010754291899502277,
+      "rewards//mean": 0.788818359375,
+      "rewards//std": 0.027611492201685905,
+      "step": 766
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1534,
+      "grad_norm": 1.1897015571594238,
+      "kl": 0.35634366050362587,
+      "learning_rate": 4.746289115992198e-06,
+      "loss": 0.0143,
+      "num_tokens": 6629192.0,
+      "reward": 0.75067138671875,
+      "reward_std": 0.006821729242801666,
+      "rewards//mean": 0.75067138671875,
+      "rewards//std": 0.023944122716784477,
+      "step": 767
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1536,
+      "grad_norm": 1.6127054691314697,
+      "kl": 0.3192887920886278,
+      "learning_rate": 4.74559221143149e-06,
+      "loss": 0.0128,
+      "num_tokens": 6637832.0,
+      "reward": 0.7333984375,
+      "reward_std": 0.00948795210570097,
+      "rewards//mean": 0.7333984375,
+      "rewards//std": 0.027524733915925026,
+      "step": 768
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1538,
+      "grad_norm": 0.9704777598381042,
+      "kl": 0.3409872241318226,
+      "learning_rate": 4.744894402346508e-06,
+      "loss": 0.0136,
+      "num_tokens": 6646552.0,
+      "reward": 0.785400390625,
+      "reward_std": 0.01048012264072895,
+      "rewards//mean": 0.785400390625,
+      "rewards//std": 0.030144404619932175,
+      "step": 769
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.154,
+      "grad_norm": 1.119908332824707,
+      "kl": 0.36791034415364265,
+      "learning_rate": 4.744195689018331e-06,
+      "loss": 0.0147,
+      "num_tokens": 6655176.0,
+      "reward": 0.7825927734375,
+      "reward_std": 0.0098853949457407,
+      "rewards//mean": 0.7825927734375,
+      "rewards//std": 0.024990104138851166,
+      "step": 770
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1542,
+      "grad_norm": 1.1021844148635864,
+      "kl": 0.36526960879564285,
+      "learning_rate": 4.743496071728396e-06,
+      "loss": 0.0146,
+      "num_tokens": 6663840.0,
+      "reward": 0.7615966796875,
+      "reward_std": 0.005251667462289333,
+      "rewards//mean": 0.7615966796875,
+      "rewards//std": 0.029023557901382446,
+      "step": 771
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1544,
+      "grad_norm": 1.5725772380828857,
+      "kl": 0.38798758387565613,
+      "learning_rate": 4.742795550758514e-06,
+      "loss": 0.0155,
+      "num_tokens": 6672440.0,
+      "reward": 0.7513427734375,
+      "reward_std": 0.005276157986372709,
+      "rewards//mean": 0.7513427734375,
+      "rewards//std": 0.020719308406114578,
+      "step": 772
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1546,
+      "grad_norm": 1.8963240385055542,
+      "kl": 0.3601706586778164,
+      "learning_rate": 4.742094126390851e-06,
+      "loss": 0.0144,
+      "num_tokens": 6680960.0,
+      "reward": 0.761962890625,
+      "reward_std": 0.007767863571643829,
+      "rewards//mean": 0.761962890625,
+      "rewards//std": 0.017892228439450264,
+      "step": 773
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1548,
+      "grad_norm": 0.9243273735046387,
+      "kl": 0.3406798876821995,
+      "learning_rate": 4.7413917989079415e-06,
+      "loss": 0.0136,
+      "num_tokens": 6689760.0,
+      "reward": 0.7685546875,
+      "reward_std": 0.008658873848617077,
+      "rewards//mean": 0.7685546875,
+      "rewards//std": 0.02934746816754341,
+      "step": 774
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.155,
+      "grad_norm": 0.9375861883163452,
+      "kl": 0.34522927552461624,
+      "learning_rate": 4.740688568592685e-06,
+      "loss": 0.0138,
+      "num_tokens": 6698392.0,
+      "reward": 0.77886962890625,
+      "reward_std": 0.008174005895853043,
+      "rewards//mean": 0.77886962890625,
+      "rewards//std": 0.021060163155198097,
+      "step": 775
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1552,
+      "grad_norm": 0.8136488199234009,
+      "kl": 0.35180309042334557,
+      "learning_rate": 4.73998443572834e-06,
+      "loss": 0.0141,
+      "num_tokens": 6707040.0,
+      "reward": 0.7554931640625,
+      "reward_std": 0.008698609657585621,
+      "rewards//mean": 0.7554931640625,
+      "rewards//std": 0.0261689480394125,
+      "step": 776
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1554,
+      "grad_norm": 0.9896641969680786,
+      "kl": 0.33184514194726944,
+      "learning_rate": 4.7392794005985324e-06,
+      "loss": 0.0133,
+      "num_tokens": 6715672.0,
+      "reward": 0.76751708984375,
+      "reward_std": 0.008659705519676208,
+      "rewards//mean": 0.76751708984375,
+      "rewards//std": 0.022086039185523987,
+      "step": 777
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1556,
+      "grad_norm": 0.8349254131317139,
+      "kl": 0.3396892622113228,
+      "learning_rate": 4.7385734634872504e-06,
+      "loss": 0.0136,
+      "num_tokens": 6724312.0,
+      "reward": 0.76275634765625,
+      "reward_std": 0.010972029529511929,
+      "rewards//mean": 0.76275634765625,
+      "rewards//std": 0.024619286879897118,
+      "step": 778
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1558,
+      "grad_norm": 0.7266840934753418,
+      "kl": 0.3403605706989765,
+      "learning_rate": 4.7378666246788444e-06,
+      "loss": 0.0136,
+      "num_tokens": 6732968.0,
+      "reward": 0.7620849609375,
+      "reward_std": 0.011202620342373848,
+      "rewards//mean": 0.7620849609375,
+      "rewards//std": 0.028896017000079155,
+      "step": 779
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.156,
+      "grad_norm": 0.6741737127304077,
+      "kl": 0.30693612433969975,
+      "learning_rate": 4.73715888445803e-06,
+      "loss": 0.0123,
+      "num_tokens": 6741616.0,
+      "reward": 0.7645263671875,
+      "reward_std": 0.01664084941148758,
+      "rewards//mean": 0.7645263671875,
+      "rewards//std": 0.028137991204857826,
+      "step": 780
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1562,
+      "grad_norm": 0.9469835162162781,
+      "kl": 0.33184751495718956,
+      "learning_rate": 4.736450243109885e-06,
+      "loss": 0.0133,
+      "num_tokens": 6750248.0,
+      "reward": 0.772216796875,
+      "reward_std": 0.010948438197374344,
+      "rewards//mean": 0.772216796875,
+      "rewards//std": 0.02752363495528698,
+      "step": 781
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1564,
+      "grad_norm": 1.3513708114624023,
+      "kl": 0.36903392523527145,
+      "learning_rate": 4.735740700919848e-06,
+      "loss": 0.0148,
+      "num_tokens": 6758784.0,
+      "reward": 0.75982666015625,
+      "reward_std": 0.011673888191580772,
+      "rewards//mean": 0.75982666015625,
+      "rewards//std": 0.02905711531639099,
+      "step": 782
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1566,
+      "grad_norm": 1.2254798412322998,
+      "kl": 0.3678886219859123,
+      "learning_rate": 4.7350302581737255e-06,
+      "loss": 0.0147,
+      "num_tokens": 6767448.0,
+      "reward": 0.7747802734375,
+      "reward_std": 0.009641564451158047,
+      "rewards//mean": 0.7747802734375,
+      "rewards//std": 0.019651401787996292,
+      "step": 783
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1568,
+      "grad_norm": 1.1070396900177002,
+      "kl": 0.3434663563966751,
+      "learning_rate": 4.734318915157682e-06,
+      "loss": 0.0137,
+      "num_tokens": 6776224.0,
+      "reward": 0.72930908203125,
+      "reward_std": 0.009073897264897823,
+      "rewards//mean": 0.72930908203125,
+      "rewards//std": 0.04494238644838333,
+      "step": 784
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.157,
+      "grad_norm": 0.925721287727356,
+      "kl": 0.3413317985832691,
+      "learning_rate": 4.7336066721582464e-06,
+      "loss": 0.0137,
+      "num_tokens": 6784832.0,
+      "reward": 0.74957275390625,
+      "reward_std": 0.008924740366637707,
+      "rewards//mean": 0.74957275390625,
+      "rewards//std": 0.03833093121647835,
+      "step": 785
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1572,
+      "grad_norm": 1.4691276550292969,
+      "kl": 0.3254700042307377,
+      "learning_rate": 4.73289352946231e-06,
+      "loss": 0.013,
+      "num_tokens": 6793464.0,
+      "reward": 0.783935546875,
+      "reward_std": 0.009136625565588474,
+      "rewards//mean": 0.783935546875,
+      "rewards//std": 0.021708672866225243,
+      "step": 786
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1574,
+      "grad_norm": 0.8711658120155334,
+      "kl": 0.34013716131448746,
+      "learning_rate": 4.732179487357127e-06,
+      "loss": 0.0136,
+      "num_tokens": 6801992.0,
+      "reward": 0.76971435546875,
+      "reward_std": 0.010332462377846241,
+      "rewards//mean": 0.76971435546875,
+      "rewards//std": 0.028943846002221107,
+      "step": 787
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1576,
+      "grad_norm": 1.2573686838150024,
+      "kl": 0.31912393122911453,
+      "learning_rate": 4.731464546130315e-06,
+      "loss": 0.0128,
+      "num_tokens": 6810688.0,
+      "reward": 0.7406005859375,
+      "reward_std": 0.01056898757815361,
+      "rewards//mean": 0.7406005859375,
+      "rewards//std": 0.026270560920238495,
+      "step": 788
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1578,
+      "grad_norm": 0.6769285798072815,
+      "kl": 0.3092736713588238,
+      "learning_rate": 4.730748706069849e-06,
+      "loss": 0.0124,
+      "num_tokens": 6819336.0,
+      "reward": 0.7562255859375,
+      "reward_std": 0.01258472166955471,
+      "rewards//mean": 0.7562255859375,
+      "rewards//std": 0.038561511784791946,
+      "step": 789
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.158,
+      "grad_norm": 0.7998493313789368,
+      "kl": 0.3514319136738777,
+      "learning_rate": 4.730031967464071e-06,
+      "loss": 0.0141,
+      "num_tokens": 6828088.0,
+      "reward": 0.74859619140625,
+      "reward_std": 0.008010081946849823,
+      "rewards//mean": 0.74859619140625,
+      "rewards//std": 0.026524242013692856,
+      "step": 790
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1582,
+      "grad_norm": 0.934973955154419,
+      "kl": 0.30601639300584793,
+      "learning_rate": 4.729314330601684e-06,
+      "loss": 0.0122,
+      "num_tokens": 6836808.0,
+      "reward": 0.757080078125,
+      "reward_std": 0.013350230641663074,
+      "rewards//mean": 0.757080078125,
+      "rewards//std": 0.020503751933574677,
+      "step": 791
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1584,
+      "grad_norm": 0.9307112097740173,
+      "kl": 0.3210490830242634,
+      "learning_rate": 4.72859579577175e-06,
+      "loss": 0.0128,
+      "num_tokens": 6845400.0,
+      "reward": 0.75616455078125,
+      "reward_std": 0.008527729660272598,
+      "rewards//mean": 0.75616455078125,
+      "rewards//std": 0.020061077550053596,
+      "step": 792
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1586,
+      "grad_norm": 1.2021913528442383,
+      "kl": 0.33378901705145836,
+      "learning_rate": 4.7278763632636974e-06,
+      "loss": 0.0134,
+      "num_tokens": 6854096.0,
+      "reward": 0.72943115234375,
+      "reward_std": 0.005128923803567886,
+      "rewards//mean": 0.72943115234375,
+      "rewards//std": 0.0317058339715004,
+      "step": 793
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1588,
+      "grad_norm": 1.0796374082565308,
+      "kl": 0.32557179778814316,
+      "learning_rate": 4.727156033367312e-06,
+      "loss": 0.013,
+      "num_tokens": 6862656.0,
+      "reward": 0.7237548828125,
+      "reward_std": 0.006958717480301857,
+      "rewards//mean": 0.7237548828125,
+      "rewards//std": 0.01773715205490589,
+      "step": 794
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.159,
+      "grad_norm": 1.2356610298156738,
+      "kl": 0.30212575383484364,
+      "learning_rate": 4.7264348063727415e-06,
+      "loss": 0.0121,
+      "num_tokens": 6871248.0,
+      "reward": 0.763671875,
+      "reward_std": 0.00585189089179039,
+      "rewards//mean": 0.763671875,
+      "rewards//std": 0.024861659854650497,
+      "step": 795
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1592,
+      "grad_norm": 1.584045648574829,
+      "kl": 0.32087236642837524,
+      "learning_rate": 4.725712682570498e-06,
+      "loss": 0.0128,
+      "num_tokens": 6879968.0,
+      "reward": 0.7618408203125,
+      "reward_std": 0.01183655858039856,
+      "rewards//mean": 0.7618408203125,
+      "rewards//std": 0.029248006641864777,
+      "step": 796
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1594,
+      "grad_norm": 1.0374621152877808,
+      "kl": 0.31318943575024605,
+      "learning_rate": 4.724989662251452e-06,
+      "loss": 0.0125,
+      "num_tokens": 6888536.0,
+      "reward": 0.76678466796875,
+      "reward_std": 0.011448527686297894,
+      "rewards//mean": 0.76678466796875,
+      "rewards//std": 0.025842614471912384,
+      "step": 797
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1596,
+      "grad_norm": 1.0205044746398926,
+      "kl": 0.30759022757411003,
+      "learning_rate": 4.724265745706837e-06,
+      "loss": 0.0123,
+      "num_tokens": 6897080.0,
+      "reward": 0.7689208984375,
+      "reward_std": 0.0078608188778162,
+      "rewards//mean": 0.7689208984375,
+      "rewards//std": 0.026348810642957687,
+      "step": 798
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1598,
+      "grad_norm": 2.31490159034729,
+      "kl": 0.3154881428927183,
+      "learning_rate": 4.723540933228245e-06,
+      "loss": 0.0126,
+      "num_tokens": 6905712.0,
+      "reward": 0.753662109375,
+      "reward_std": 0.0058388663455843925,
+      "rewards//mean": 0.753662109375,
+      "rewards//std": 0.029780646786093712,
+      "step": 799
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.16,
+      "grad_norm": 1.1097886562347412,
+      "kl": 0.3300345875322819,
+      "learning_rate": 4.7228152251076295e-06,
+      "loss": 0.0132,
+      "num_tokens": 6914304.0,
+      "reward": 0.77337646484375,
+      "reward_std": 0.009435447864234447,
+      "rewards//mean": 0.77337646484375,
+      "rewards//std": 0.024680698290467262,
+      "step": 800
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1602,
+      "grad_norm": 1.193706750869751,
+      "kl": 0.3689091205596924,
+      "learning_rate": 4.7220886216373095e-06,
+      "loss": 0.0148,
+      "num_tokens": 6922960.0,
+      "reward": 0.78070068359375,
+      "reward_std": 0.007954924367368221,
+      "rewards//mean": 0.78070068359375,
+      "rewards//std": 0.030088046565651894,
+      "step": 801
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1604,
+      "grad_norm": 1.4976460933685303,
+      "kl": 0.34658120200037956,
+      "learning_rate": 4.7213611231099575e-06,
+      "loss": 0.0139,
+      "num_tokens": 6931632.0,
+      "reward": 0.7415771484375,
+      "reward_std": 0.007033228408545256,
+      "rewards//mean": 0.7415771484375,
+      "rewards//std": 0.03241577744483948,
+      "step": 802
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1606,
+      "grad_norm": 1.3477445840835571,
+      "kl": 0.35779566690325737,
+      "learning_rate": 4.7206327298186105e-06,
+      "loss": 0.0143,
+      "num_tokens": 6940272.0,
+      "reward": 0.77032470703125,
+      "reward_std": 0.006013727746903896,
+      "rewards//mean": 0.77032470703125,
+      "rewards//std": 0.021487809717655182,
+      "step": 803
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1608,
+      "grad_norm": 2.233660936355591,
+      "kl": 0.324648205190897,
+      "learning_rate": 4.7199034420566656e-06,
+      "loss": 0.013,
+      "num_tokens": 6948856.0,
+      "reward": 0.74658203125,
+      "reward_std": 0.009060757234692574,
+      "rewards//mean": 0.74658203125,
+      "rewards//std": 0.033411283046007156,
+      "step": 804
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.161,
+      "grad_norm": 1.2979011535644531,
+      "kl": 0.35064204037189484,
+      "learning_rate": 4.7191732601178795e-06,
+      "loss": 0.014,
+      "num_tokens": 6957592.0,
+      "reward": 0.7510986328125,
+      "reward_std": 0.007960843853652477,
+      "rewards//mean": 0.7510986328125,
+      "rewards//std": 0.027572816237807274,
+      "step": 805
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1612,
+      "grad_norm": 1.3037084341049194,
+      "kl": 0.37102117761969566,
+      "learning_rate": 4.71844218429637e-06,
+      "loss": 0.0148,
+      "num_tokens": 6966240.0,
+      "reward": 0.77691650390625,
+      "reward_std": 0.013105214573442936,
+      "rewards//mean": 0.77691650390625,
+      "rewards//std": 0.0265219584107399,
+      "step": 806
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1614,
+      "grad_norm": 1.3451786041259766,
+      "kl": 0.3336266949772835,
+      "learning_rate": 4.717710214886614e-06,
+      "loss": 0.0133,
+      "num_tokens": 6974904.0,
+      "reward": 0.7286376953125,
+      "reward_std": 0.009902372024953365,
+      "rewards//mean": 0.7286376953125,
+      "rewards//std": 0.03538607805967331,
+      "step": 807
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1616,
+      "grad_norm": 1.0671733617782593,
+      "kl": 0.3264966905117035,
+      "learning_rate": 4.716977352183449e-06,
+      "loss": 0.0131,
+      "num_tokens": 6983504.0,
+      "reward": 0.773193359375,
+      "reward_std": 0.007287709973752499,
+      "rewards//mean": 0.773193359375,
+      "rewards//std": 0.025993958115577698,
+      "step": 808
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1618,
+      "grad_norm": 1.8360722064971924,
+      "kl": 0.33740856125950813,
+      "learning_rate": 4.716243596482071e-06,
+      "loss": 0.0135,
+      "num_tokens": 6992216.0,
+      "reward": 0.75079345703125,
+      "reward_std": 0.008213981986045837,
+      "rewards//mean": 0.75079345703125,
+      "rewards//std": 0.028178511187434196,
+      "step": 809
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.162,
+      "grad_norm": 1.5714982748031616,
+      "kl": 0.3707769885659218,
+      "learning_rate": 4.715508948078037e-06,
+      "loss": 0.0148,
+      "num_tokens": 7000816.0,
+      "reward": 0.77569580078125,
+      "reward_std": 0.012640546075999737,
+      "rewards//mean": 0.77569580078125,
+      "rewards//std": 0.034434448927640915,
+      "step": 810
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1622,
+      "grad_norm": 1.2873075008392334,
+      "kl": 0.3405948132276535,
+      "learning_rate": 4.714773407267264e-06,
+      "loss": 0.0136,
+      "num_tokens": 7009480.0,
+      "reward": 0.7650146484375,
+      "reward_std": 0.006831423845142126,
+      "rewards//mean": 0.7650146484375,
+      "rewards//std": 0.020054001361131668,
+      "step": 811
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1624,
+      "grad_norm": 0.8567954301834106,
+      "kl": 0.3825432136654854,
+      "learning_rate": 4.714036974346028e-06,
+      "loss": 0.0153,
+      "num_tokens": 7018136.0,
+      "reward": 0.7357177734375,
+      "reward_std": 0.011918211355805397,
+      "rewards//mean": 0.7357177734375,
+      "rewards//std": 0.028866665437817574,
+      "step": 812
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1626,
+      "grad_norm": 1.2704569101333618,
+      "kl": 0.35488636791706085,
+      "learning_rate": 4.7132996496109625e-06,
+      "loss": 0.0142,
+      "num_tokens": 7026824.0,
+      "reward": 0.76324462890625,
+      "reward_std": 0.010413013398647308,
+      "rewards//mean": 0.76324462890625,
+      "rewards//std": 0.025464370846748352,
+      "step": 813
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1628,
+      "grad_norm": 1.061576247215271,
+      "kl": 0.3901236914098263,
+      "learning_rate": 4.712561433359064e-06,
+      "loss": 0.0156,
+      "num_tokens": 7035488.0,
+      "reward": 0.74591064453125,
+      "reward_std": 0.008115001022815704,
+      "rewards//mean": 0.74591064453125,
+      "rewards//std": 0.02554805390536785,
+      "step": 814
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.163,
+      "grad_norm": 1.8980876207351685,
+      "kl": 0.3521212972700596,
+      "learning_rate": 4.7118223258876845e-06,
+      "loss": 0.0141,
+      "num_tokens": 7044072.0,
+      "reward": 0.75384521484375,
+      "reward_std": 0.007846582680940628,
+      "rewards//mean": 0.75384521484375,
+      "rewards//std": 0.016912657767534256,
+      "step": 815
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1632,
+      "grad_norm": 1.1933066844940186,
+      "kl": 0.38137778267264366,
+      "learning_rate": 4.711082327494536e-06,
+      "loss": 0.0153,
+      "num_tokens": 7052744.0,
+      "reward": 0.76531982421875,
+      "reward_std": 0.007578435353934765,
+      "rewards//mean": 0.76531982421875,
+      "rewards//std": 0.02658751606941223,
+      "step": 816
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1634,
+      "grad_norm": 1.466513991355896,
+      "kl": 0.4100740812718868,
+      "learning_rate": 4.710341438477691e-06,
+      "loss": 0.0164,
+      "num_tokens": 7061384.0,
+      "reward": 0.73834228515625,
+      "reward_std": 0.010529394261538982,
+      "rewards//mean": 0.73834228515625,
+      "rewards//std": 0.03200426325201988,
+      "step": 817
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1636,
+      "grad_norm": 1.2618350982666016,
+      "kl": 0.39261148124933243,
+      "learning_rate": 4.709599659135579e-06,
+      "loss": 0.0157,
+      "num_tokens": 7070008.0,
+      "reward": 0.73333740234375,
+      "reward_std": 0.009211428463459015,
+      "rewards//mean": 0.73333740234375,
+      "rewards//std": 0.03585033491253853,
+      "step": 818
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1638,
+      "grad_norm": 1.817918062210083,
+      "kl": 0.37963099777698517,
+      "learning_rate": 4.708856989766988e-06,
+      "loss": 0.0152,
+      "num_tokens": 7078680.0,
+      "reward": 0.78253173828125,
+      "reward_std": 0.008948778733611107,
+      "rewards//mean": 0.78253173828125,
+      "rewards//std": 0.018895337358117104,
+      "step": 819
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.164,
+      "grad_norm": 1.1052753925323486,
+      "kl": 0.3860554024577141,
+      "learning_rate": 4.708113430671066e-06,
+      "loss": 0.0154,
+      "num_tokens": 7087336.0,
+      "reward": 0.73651123046875,
+      "reward_std": 0.007864532060921192,
+      "rewards//mean": 0.73651123046875,
+      "rewards//std": 0.027699550613760948,
+      "step": 820
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1642,
+      "grad_norm": 1.0943959951400757,
+      "kl": 0.4198205694556236,
+      "learning_rate": 4.707368982147318e-06,
+      "loss": 0.0168,
+      "num_tokens": 7096016.0,
+      "reward": 0.75958251953125,
+      "reward_std": 0.008250880986452103,
+      "rewards//mean": 0.75958251953125,
+      "rewards//std": 0.024159371852874756,
+      "step": 821
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1644,
+      "grad_norm": 1.2998090982437134,
+      "kl": 0.4261195734143257,
+      "learning_rate": 4.706623644495608e-06,
+      "loss": 0.017,
+      "num_tokens": 7104736.0,
+      "reward": 0.71905517578125,
+      "reward_std": 0.007329055108129978,
+      "rewards//mean": 0.71905517578125,
+      "rewards//std": 0.03233412653207779,
+      "step": 822
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1646,
+      "grad_norm": 1.3837676048278809,
+      "kl": 0.416591789573431,
+      "learning_rate": 4.705877418016157e-06,
+      "loss": 0.0167,
+      "num_tokens": 7113368.0,
+      "reward": 0.7564697265625,
+      "reward_std": 0.010825035162270069,
+      "rewards//mean": 0.7564697265625,
+      "rewards//std": 0.029000600799918175,
+      "step": 823
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1648,
+      "grad_norm": 1.2816661596298218,
+      "kl": 0.44941718876361847,
+      "learning_rate": 4.705130303009547e-06,
+      "loss": 0.018,
+      "num_tokens": 7122128.0,
+      "reward": 0.77276611328125,
+      "reward_std": 0.008830229751765728,
+      "rewards//mean": 0.77276611328125,
+      "rewards//std": 0.021638043224811554,
+      "step": 824
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.165,
+      "grad_norm": 1.167736530303955,
+      "kl": 0.41115984693169594,
+      "learning_rate": 4.7043822997767145e-06,
+      "loss": 0.0164,
+      "num_tokens": 7130776.0,
+      "reward": 0.7608642578125,
+      "reward_std": 0.007697493769228458,
+      "rewards//mean": 0.7608642578125,
+      "rewards//std": 0.014951195567846298,
+      "step": 825
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1652,
+      "grad_norm": 1.9024914503097534,
+      "kl": 0.44243358448147774,
+      "learning_rate": 4.703633408618955e-06,
+      "loss": 0.0177,
+      "num_tokens": 7139528.0,
+      "reward": 0.739990234375,
+      "reward_std": 0.012601775117218494,
+      "rewards//mean": 0.739990234375,
+      "rewards//std": 0.02922237664461136,
+      "step": 826
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1654,
+      "grad_norm": 1.651120901107788,
+      "kl": 0.41875192895531654,
+      "learning_rate": 4.702883629837922e-06,
+      "loss": 0.0168,
+      "num_tokens": 7148128.0,
+      "reward": 0.71734619140625,
+      "reward_std": 0.008713518269360065,
+      "rewards//mean": 0.71734619140625,
+      "rewards//std": 0.04558815062046051,
+      "step": 827
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1656,
+      "grad_norm": 3.4639272689819336,
+      "kl": 0.44034529104828835,
+      "learning_rate": 4.7021329637356274e-06,
+      "loss": 0.0176,
+      "num_tokens": 7156728.0,
+      "reward": 0.72979736328125,
+      "reward_std": 0.007449123077094555,
+      "rewards//mean": 0.72979736328125,
+      "rewards//std": 0.03378652408719063,
+      "step": 828
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1658,
+      "grad_norm": 1.6177629232406616,
+      "kl": 0.39477837830781937,
+      "learning_rate": 4.701381410614437e-06,
+      "loss": 0.0158,
+      "num_tokens": 7165408.0,
+      "reward": 0.76959228515625,
+      "reward_std": 0.01015202235430479,
+      "rewards//mean": 0.76959228515625,
+      "rewards//std": 0.029647106304764748,
+      "step": 829
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.166,
+      "grad_norm": 1.7870838642120361,
+      "kl": 0.49362414330244064,
+      "learning_rate": 4.700628970777078e-06,
+      "loss": 0.0197,
+      "num_tokens": 7174016.0,
+      "reward": 0.759765625,
+      "reward_std": 0.011450910940766335,
+      "rewards//mean": 0.759765625,
+      "rewards//std": 0.033861320465803146,
+      "step": 830
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1662,
+      "grad_norm": 1.3444162607192993,
+      "kl": 0.45138998702168465,
+      "learning_rate": 4.699875644526633e-06,
+      "loss": 0.0181,
+      "num_tokens": 7182656.0,
+      "reward": 0.752685546875,
+      "reward_std": 0.009241987019777298,
+      "rewards//mean": 0.752685546875,
+      "rewards//std": 0.02500615082681179,
+      "step": 831
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1664,
+      "grad_norm": 2.9015026092529297,
+      "kl": 0.45517536252737045,
+      "learning_rate": 4.699121432166542e-06,
+      "loss": 0.0182,
+      "num_tokens": 7191360.0,
+      "reward": 0.7528076171875,
+      "reward_std": 0.005533740855753422,
+      "rewards//mean": 0.7528076171875,
+      "rewards//std": 0.033112503588199615,
+      "step": 832
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1666,
+      "grad_norm": 1.1061749458312988,
+      "kl": 0.5402024127542973,
+      "learning_rate": 4.6983663340006e-06,
+      "loss": 0.0216,
+      "num_tokens": 7200080.0,
+      "reward": 0.75238037109375,
+      "reward_std": 0.009564736858010292,
+      "rewards//mean": 0.75238037109375,
+      "rewards//std": 0.019030246883630753,
+      "step": 833
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1668,
+      "grad_norm": 0.9863348603248596,
+      "kl": 0.4983534514904022,
+      "learning_rate": 4.697610350332962e-06,
+      "loss": 0.0199,
+      "num_tokens": 7208816.0,
+      "reward": 0.75982666015625,
+      "reward_std": 0.010304231196641922,
+      "rewards//mean": 0.75982666015625,
+      "rewards//std": 0.026097338646650314,
+      "step": 834
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.167,
+      "grad_norm": 0.8810539841651917,
+      "kl": 0.4796448089182377,
+      "learning_rate": 4.696853481468137e-06,
+      "loss": 0.0192,
+      "num_tokens": 7217488.0,
+      "reward": 0.75048828125,
+      "reward_std": 0.009355641901493073,
+      "rewards//mean": 0.75048828125,
+      "rewards//std": 0.030659254640340805,
+      "step": 835
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1672,
+      "grad_norm": 1.3327339887619019,
+      "kl": 0.48946595191955566,
+      "learning_rate": 4.6960957277109945e-06,
+      "loss": 0.0196,
+      "num_tokens": 7226024.0,
+      "reward": 0.75537109375,
+      "reward_std": 0.009866978041827679,
+      "rewards//mean": 0.75537109375,
+      "rewards//std": 0.025596076622605324,
+      "step": 836
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1674,
+      "grad_norm": 0.9784926772117615,
+      "kl": 0.5766041316092014,
+      "learning_rate": 4.695337089366754e-06,
+      "loss": 0.0231,
+      "num_tokens": 7234672.0,
+      "reward": 0.74761962890625,
+      "reward_std": 0.012241121381521225,
+      "rewards//mean": 0.74761962890625,
+      "rewards//std": 0.03139396011829376,
+      "step": 837
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1676,
+      "grad_norm": 1.5746252536773682,
+      "kl": 0.528806246817112,
+      "learning_rate": 4.694577566740996e-06,
+      "loss": 0.0212,
+      "num_tokens": 7243248.0,
+      "reward": 0.72967529296875,
+      "reward_std": 0.00966464914381504,
+      "rewards//mean": 0.72967529296875,
+      "rewards//std": 0.029709843918681145,
+      "step": 838
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1678,
+      "grad_norm": 1.091158390045166,
+      "kl": 0.5902933292090893,
+      "learning_rate": 4.693817160139657e-06,
+      "loss": 0.0236,
+      "num_tokens": 7252024.0,
+      "reward": 0.73333740234375,
+      "reward_std": 0.008134350180625916,
+      "rewards//mean": 0.73333740234375,
+      "rewards//std": 0.02597873844206333,
+      "step": 839
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.168,
+      "grad_norm": 1.2695715427398682,
+      "kl": 0.5501630380749702,
+      "learning_rate": 4.693055869869029e-06,
+      "loss": 0.022,
+      "num_tokens": 7260632.0,
+      "reward": 0.74896240234375,
+      "reward_std": 0.007687639910727739,
+      "rewards//mean": 0.74896240234375,
+      "rewards//std": 0.0311431884765625,
+      "step": 840
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1682,
+      "grad_norm": 2.8090455532073975,
+      "kl": 0.5814741887152195,
+      "learning_rate": 4.692293696235758e-06,
+      "loss": 0.0233,
+      "num_tokens": 7269232.0,
+      "reward": 0.76068115234375,
+      "reward_std": 0.006853449624031782,
+      "rewards//mean": 0.76068115234375,
+      "rewards//std": 0.019774554297327995,
+      "step": 841
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1684,
+      "grad_norm": 0.8583130836486816,
+      "kl": 0.5908543989062309,
+      "learning_rate": 4.6915306395468485e-06,
+      "loss": 0.0236,
+      "num_tokens": 7277888.0,
+      "reward": 0.75555419921875,
+      "reward_std": 0.009005377069115639,
+      "rewards//mean": 0.75555419921875,
+      "rewards//std": 0.018253128975629807,
+      "step": 842
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1686,
+      "grad_norm": 1.7500971555709839,
+      "kl": 0.5774266049265862,
+      "learning_rate": 4.690766700109659e-06,
+      "loss": 0.0231,
+      "num_tokens": 7286528.0,
+      "reward": 0.7679443359375,
+      "reward_std": 0.010070763528347015,
+      "rewards//mean": 0.7679443359375,
+      "rewards//std": 0.02478085085749626,
+      "step": 843
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1688,
+      "grad_norm": 1.1226806640625,
+      "kl": 0.5556729286909103,
+      "learning_rate": 4.690001878231906e-06,
+      "loss": 0.0222,
+      "num_tokens": 7295168.0,
+      "reward": 0.74505615234375,
+      "reward_std": 0.009786233305931091,
+      "rewards//mean": 0.74505615234375,
+      "rewards//std": 0.01919025555253029,
+      "step": 844
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.169,
+      "grad_norm": 1.5926430225372314,
+      "kl": 0.5301719754934311,
+      "learning_rate": 4.689236174221658e-06,
+      "loss": 0.0212,
+      "num_tokens": 7303800.0,
+      "reward": 0.73565673828125,
+      "reward_std": 0.010708114132285118,
+      "rewards//mean": 0.73565673828125,
+      "rewards//std": 0.017935220152139664,
+      "step": 845
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1692,
+      "grad_norm": 1.3184345960617065,
+      "kl": 0.5430991984903812,
+      "learning_rate": 4.688469588387339e-06,
+      "loss": 0.0217,
+      "num_tokens": 7312504.0,
+      "reward": 0.77593994140625,
+      "reward_std": 0.009601364843547344,
+      "rewards//mean": 0.77593994140625,
+      "rewards//std": 0.026642685756087303,
+      "step": 846
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1694,
+      "grad_norm": 0.8678821325302124,
+      "kl": 0.590129129588604,
+      "learning_rate": 4.687702121037734e-06,
+      "loss": 0.0236,
+      "num_tokens": 7321176.0,
+      "reward": 0.740234375,
+      "reward_std": 0.009357824921607971,
+      "rewards//mean": 0.740234375,
+      "rewards//std": 0.026721050962805748,
+      "step": 847
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1696,
+      "grad_norm": 3.635012149810791,
+      "kl": 0.5143630467355251,
+      "learning_rate": 4.6869337724819745e-06,
+      "loss": 0.0206,
+      "num_tokens": 7329792.0,
+      "reward": 0.7623291015625,
+      "reward_std": 0.010166186839342117,
+      "rewards//mean": 0.7623291015625,
+      "rewards//std": 0.02383934147655964,
+      "step": 848
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1698,
+      "grad_norm": 1.2785440683364868,
+      "kl": 0.5269539132714272,
+      "learning_rate": 4.686164543029554e-06,
+      "loss": 0.0211,
+      "num_tokens": 7338440.0,
+      "reward": 0.774169921875,
+      "reward_std": 0.008573425933718681,
+      "rewards//mean": 0.774169921875,
+      "rewards//std": 0.02073865942656994,
+      "step": 849
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.17,
+      "grad_norm": 1.50819730758667,
+      "kl": 0.49373095482587814,
+      "learning_rate": 4.685394432990316e-06,
+      "loss": 0.0197,
+      "num_tokens": 7347016.0,
+      "reward": 0.756591796875,
+      "reward_std": 0.011000225320458412,
+      "rewards//mean": 0.756591796875,
+      "rewards//std": 0.027347072958946228,
+      "step": 850
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1702,
+      "grad_norm": 0.9949831962585449,
+      "kl": 0.5279292948544025,
+      "learning_rate": 4.684623442674463e-06,
+      "loss": 0.0211,
+      "num_tokens": 7355664.0,
+      "reward": 0.69683837890625,
+      "reward_std": 0.0144025394693017,
+      "rewards//mean": 0.69683837890625,
+      "rewards//std": 0.041202642023563385,
+      "step": 851
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1704,
+      "grad_norm": 1.1295602321624756,
+      "kl": 0.5627134665846825,
+      "learning_rate": 4.683851572392548e-06,
+      "loss": 0.0225,
+      "num_tokens": 7364392.0,
+      "reward": 0.7559814453125,
+      "reward_std": 0.01152932457625866,
+      "rewards//mean": 0.7559814453125,
+      "rewards//std": 0.03178766742348671,
+      "step": 852
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1706,
+      "grad_norm": 1.0643706321716309,
+      "kl": 0.5173224434256554,
+      "learning_rate": 4.68307882245548e-06,
+      "loss": 0.0207,
+      "num_tokens": 7373032.0,
+      "reward": 0.7471923828125,
+      "reward_std": 0.010553266853094101,
+      "rewards//mean": 0.7471923828125,
+      "rewards//std": 0.03074578382074833,
+      "step": 853
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1708,
+      "grad_norm": 0.8686037063598633,
+      "kl": 0.5057759210467339,
+      "learning_rate": 4.682305193174524e-06,
+      "loss": 0.0202,
+      "num_tokens": 7381688.0,
+      "reward": 0.75091552734375,
+      "reward_std": 0.014319094829261303,
+      "rewards//mean": 0.75091552734375,
+      "rewards//std": 0.027645394206047058,
+      "step": 854
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.171,
+      "grad_norm": 2.133981943130493,
+      "kl": 0.5340125001966953,
+      "learning_rate": 4.681530684861298e-06,
+      "loss": 0.0214,
+      "num_tokens": 7390344.0,
+      "reward": 0.77490234375,
+      "reward_std": 0.012180844321846962,
+      "rewards//mean": 0.77490234375,
+      "rewards//std": 0.020534737035632133,
+      "step": 855
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1712,
+      "grad_norm": 1.4045356512069702,
+      "kl": 0.5420391000807285,
+      "learning_rate": 4.680755297827772e-06,
+      "loss": 0.0217,
+      "num_tokens": 7398960.0,
+      "reward": 0.76593017578125,
+      "reward_std": 0.007923927158117294,
+      "rewards//mean": 0.76593017578125,
+      "rewards//std": 0.02088331989943981,
+      "step": 856
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1714,
+      "grad_norm": 0.8970532417297363,
+      "kl": 0.49403341114521027,
+      "learning_rate": 4.6799790323862735e-06,
+      "loss": 0.0198,
+      "num_tokens": 7407632.0,
+      "reward": 0.7232666015625,
+      "reward_std": 0.01003449596464634,
+      "rewards//mean": 0.7232666015625,
+      "rewards//std": 0.029150541871786118,
+      "step": 857
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1716,
+      "grad_norm": 1.2930840253829956,
+      "kl": 0.4648851417005062,
+      "learning_rate": 4.679201888849481e-06,
+      "loss": 0.0186,
+      "num_tokens": 7416224.0,
+      "reward": 0.77532958984375,
+      "reward_std": 0.008281329646706581,
+      "rewards//mean": 0.77532958984375,
+      "rewards//std": 0.02078668773174286,
+      "step": 858
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1718,
+      "grad_norm": 0.9651484489440918,
+      "kl": 0.44804292544722557,
+      "learning_rate": 4.678423867530428e-06,
+      "loss": 0.0179,
+      "num_tokens": 7424872.0,
+      "reward": 0.77166748046875,
+      "reward_std": 0.009435847401618958,
+      "rewards//mean": 0.77166748046875,
+      "rewards//std": 0.024149343371391296,
+      "step": 859
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.172,
+      "grad_norm": 1.372725486755371,
+      "kl": 0.4652128331363201,
+      "learning_rate": 4.677644968742503e-06,
+      "loss": 0.0186,
+      "num_tokens": 7433544.0,
+      "reward": 0.77142333984375,
+      "reward_std": 0.011212656274437904,
+      "rewards//mean": 0.77142333984375,
+      "rewards//std": 0.028442122042179108,
+      "step": 860
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1722,
+      "grad_norm": 1.1838834285736084,
+      "kl": 0.4007790870964527,
+      "learning_rate": 4.676865192799443e-06,
+      "loss": 0.016,
+      "num_tokens": 7442144.0,
+      "reward": 0.75579833984375,
+      "reward_std": 0.008447399362921715,
+      "rewards//mean": 0.75579833984375,
+      "rewards//std": 0.024019237607717514,
+      "step": 861
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1724,
+      "grad_norm": 0.6647836565971375,
+      "kl": 0.45183153450489044,
+      "learning_rate": 4.676084540015345e-06,
+      "loss": 0.0181,
+      "num_tokens": 7450776.0,
+      "reward": 0.75628662109375,
+      "reward_std": 0.009683351032435894,
+      "rewards//mean": 0.75628662109375,
+      "rewards//std": 0.026084575802087784,
+      "step": 862
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1726,
+      "grad_norm": 1.740395426750183,
+      "kl": 0.4510505050420761,
+      "learning_rate": 4.675303010704654e-06,
+      "loss": 0.018,
+      "num_tokens": 7459440.0,
+      "reward": 0.7762451171875,
+      "reward_std": 0.01001917663961649,
+      "rewards//mean": 0.7762451171875,
+      "rewards//std": 0.022178800776600838,
+      "step": 863
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1728,
+      "grad_norm": 0.8341839909553528,
+      "kl": 0.42043209448456764,
+      "learning_rate": 4.674520605182171e-06,
+      "loss": 0.0168,
+      "num_tokens": 7468160.0,
+      "reward": 0.7720947265625,
+      "reward_std": 0.010514896363019943,
+      "rewards//mean": 0.7720947265625,
+      "rewards//std": 0.02007814310491085,
+      "step": 864
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.173,
+      "grad_norm": 0.6752360463142395,
+      "kl": 0.43009642139077187,
+      "learning_rate": 4.673737323763048e-06,
+      "loss": 0.0172,
+      "num_tokens": 7476752.0,
+      "reward": 0.76568603515625,
+      "reward_std": 0.00976257212460041,
+      "rewards//mean": 0.76568603515625,
+      "rewards//std": 0.015445588156580925,
+      "step": 865
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1732,
+      "grad_norm": 0.6788051128387451,
+      "kl": 0.4369927644729614,
+      "learning_rate": 4.672953166762791e-06,
+      "loss": 0.0175,
+      "num_tokens": 7485448.0,
+      "reward": 0.76446533203125,
+      "reward_std": 0.012529331259429455,
+      "rewards//mean": 0.76446533203125,
+      "rewards//std": 0.025926828384399414,
+      "step": 866
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1734,
+      "grad_norm": 0.6571235060691833,
+      "kl": 0.45430251583456993,
+      "learning_rate": 4.672168134497258e-06,
+      "loss": 0.0182,
+      "num_tokens": 7494104.0,
+      "reward": 0.720947265625,
+      "reward_std": 0.015459833666682243,
+      "rewards//mean": 0.720947265625,
+      "rewards//std": 0.031657155603170395,
+      "step": 867
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1736,
+      "grad_norm": 0.6456576585769653,
+      "kl": 0.4099312573671341,
+      "learning_rate": 4.671382227282661e-06,
+      "loss": 0.0164,
+      "num_tokens": 7502752.0,
+      "reward": 0.75372314453125,
+      "reward_std": 0.011794820427894592,
+      "rewards//mean": 0.75372314453125,
+      "rewards//std": 0.02672494389116764,
+      "step": 868
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1738,
+      "grad_norm": 0.8506948947906494,
+      "kl": 0.41736486181616783,
+      "learning_rate": 4.670595445435561e-06,
+      "loss": 0.0167,
+      "num_tokens": 7511400.0,
+      "reward": 0.7132568359375,
+      "reward_std": 0.010492125526070595,
+      "rewards//mean": 0.7132568359375,
+      "rewards//std": 0.03419385850429535,
+      "step": 869
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.174,
+      "grad_norm": 0.5586651563644409,
+      "kl": 0.40891773998737335,
+      "learning_rate": 4.669807789272877e-06,
+      "loss": 0.0164,
+      "num_tokens": 7520040.0,
+      "reward": 0.76055908203125,
+      "reward_std": 0.009399401023983955,
+      "rewards//mean": 0.76055908203125,
+      "rewards//std": 0.025711068883538246,
+      "step": 870
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1742,
+      "grad_norm": 0.8430010080337524,
+      "kl": 0.37954793870449066,
+      "learning_rate": 4.669019259111873e-06,
+      "loss": 0.0152,
+      "num_tokens": 7528672.0,
+      "reward": 0.77191162109375,
+      "reward_std": 0.011711275205016136,
+      "rewards//mean": 0.77191162109375,
+      "rewards//std": 0.022103851661086082,
+      "step": 871
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1744,
+      "grad_norm": 0.6705936789512634,
+      "kl": 0.3731926344335079,
+      "learning_rate": 4.668229855270172e-06,
+      "loss": 0.0149,
+      "num_tokens": 7537480.0,
+      "reward": 0.76348876953125,
+      "reward_std": 0.017561282962560654,
+      "rewards//mean": 0.76348876953125,
+      "rewards//std": 0.03937086835503578,
+      "step": 872
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1746,
+      "grad_norm": 0.5107825994491577,
+      "kl": 0.3830333612859249,
+      "learning_rate": 4.667439578065745e-06,
+      "loss": 0.0153,
+      "num_tokens": 7546136.0,
+      "reward": 0.76861572265625,
+      "reward_std": 0.008930927142500877,
+      "rewards//mean": 0.76861572265625,
+      "rewards//std": 0.0147271528840065,
+      "step": 873
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1748,
+      "grad_norm": 0.6749988794326782,
+      "kl": 0.32537274435162544,
+      "learning_rate": 4.666648427816914e-06,
+      "loss": 0.013,
+      "num_tokens": 7554760.0,
+      "reward": 0.7623291015625,
+      "reward_std": 0.014128495939075947,
+      "rewards//mean": 0.7623291015625,
+      "rewards//std": 0.0285969115793705,
+      "step": 874
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.175,
+      "grad_norm": 0.5768765807151794,
+      "kl": 0.3486756831407547,
+      "learning_rate": 4.665856404842356e-06,
+      "loss": 0.0139,
+      "num_tokens": 7563360.0,
+      "reward": 0.76446533203125,
+      "reward_std": 0.010931688360869884,
+      "rewards//mean": 0.76446533203125,
+      "rewards//std": 0.028379783034324646,
+      "step": 875
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1752,
+      "grad_norm": 0.5427371859550476,
+      "kl": 0.3550872132182121,
+      "learning_rate": 4.665063509461098e-06,
+      "loss": 0.0142,
+      "num_tokens": 7572032.0,
+      "reward": 0.74072265625,
+      "reward_std": 0.008837481960654259,
+      "rewards//mean": 0.74072265625,
+      "rewards//std": 0.029343342408537865,
+      "step": 876
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1754,
+      "grad_norm": 0.607874870300293,
+      "kl": 0.3369791731238365,
+      "learning_rate": 4.664269741992516e-06,
+      "loss": 0.0135,
+      "num_tokens": 7580640.0,
+      "reward": 0.73492431640625,
+      "reward_std": 0.009598618373274803,
+      "rewards//mean": 0.73492431640625,
+      "rewards//std": 0.032740283757448196,
+      "step": 877
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1756,
+      "grad_norm": 0.5573941469192505,
+      "kl": 0.33512082695961,
+      "learning_rate": 4.663475102756341e-06,
+      "loss": 0.0134,
+      "num_tokens": 7589144.0,
+      "reward": 0.721923828125,
+      "reward_std": 0.008135411888360977,
+      "rewards//mean": 0.721923828125,
+      "rewards//std": 0.031764086335897446,
+      "step": 878
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1758,
+      "grad_norm": 0.5206130146980286,
+      "kl": 0.33247267454862595,
+      "learning_rate": 4.662679592072653e-06,
+      "loss": 0.0133,
+      "num_tokens": 7597784.0,
+      "reward": 0.738037109375,
+      "reward_std": 0.011560275219380856,
+      "rewards//mean": 0.738037109375,
+      "rewards//std": 0.02845815010368824,
+      "step": 879
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.176,
+      "grad_norm": 0.4959293305873871,
+      "kl": 0.3310950994491577,
+      "learning_rate": 4.661883210261884e-06,
+      "loss": 0.0132,
+      "num_tokens": 7606448.0,
+      "reward": 0.74853515625,
+      "reward_std": 0.009194498881697655,
+      "rewards//mean": 0.74853515625,
+      "rewards//std": 0.024185124784708023,
+      "step": 880
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1762,
+      "grad_norm": 0.5584758520126343,
+      "kl": 0.31146182119846344,
+      "learning_rate": 4.661085957644817e-06,
+      "loss": 0.0125,
+      "num_tokens": 7615096.0,
+      "reward": 0.7435302734375,
+      "reward_std": 0.011724255979061127,
+      "rewards//mean": 0.7435302734375,
+      "rewards//std": 0.02828180231153965,
+      "step": 881
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1764,
+      "grad_norm": 0.5160636901855469,
+      "kl": 0.31183508411049843,
+      "learning_rate": 4.660287834542585e-06,
+      "loss": 0.0125,
+      "num_tokens": 7623656.0,
+      "reward": 0.75146484375,
+      "reward_std": 0.009842973202466965,
+      "rewards//mean": 0.75146484375,
+      "rewards//std": 0.026406485587358475,
+      "step": 882
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1766,
+      "grad_norm": 0.5011554956436157,
+      "kl": 0.27698101848363876,
+      "learning_rate": 4.659488841276671e-06,
+      "loss": 0.0111,
+      "num_tokens": 7632288.0,
+      "reward": 0.73583984375,
+      "reward_std": 0.00816989317536354,
+      "rewards//mean": 0.73583984375,
+      "rewards//std": 0.03832703083753586,
+      "step": 883
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1768,
+      "grad_norm": 0.5525965690612793,
+      "kl": 0.3069544155150652,
+      "learning_rate": 4.65868897816891e-06,
+      "loss": 0.0123,
+      "num_tokens": 7640848.0,
+      "reward": 0.71710205078125,
+      "reward_std": 0.008834538981318474,
+      "rewards//mean": 0.71710205078125,
+      "rewards//std": 0.024956993758678436,
+      "step": 884
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.177,
+      "grad_norm": 0.7361370325088501,
+      "kl": 0.3224721159785986,
+      "learning_rate": 4.6578882455414865e-06,
+      "loss": 0.0129,
+      "num_tokens": 7649536.0,
+      "reward": 0.72198486328125,
+      "reward_std": 0.01567103900015354,
+      "rewards//mean": 0.72198486328125,
+      "rewards//std": 0.03261938691139221,
+      "step": 885
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1772,
+      "grad_norm": 0.5023349523544312,
+      "kl": 0.29884358681738377,
+      "learning_rate": 4.657086643716937e-06,
+      "loss": 0.012,
+      "num_tokens": 7658176.0,
+      "reward": 0.762939453125,
+      "reward_std": 0.007250561378896236,
+      "rewards//mean": 0.762939453125,
+      "rewards//std": 0.022850144654512405,
+      "step": 886
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1774,
+      "grad_norm": 0.5194371342658997,
+      "kl": 0.28458988294005394,
+      "learning_rate": 4.656284173018144e-06,
+      "loss": 0.0114,
+      "num_tokens": 7666712.0,
+      "reward": 0.76953125,
+      "reward_std": 0.01195722445845604,
+      "rewards//mean": 0.76953125,
+      "rewards//std": 0.024160075932741165,
+      "step": 887
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1776,
+      "grad_norm": 0.46291840076446533,
+      "kl": 0.281252883374691,
+      "learning_rate": 4.655480833768344e-06,
+      "loss": 0.0113,
+      "num_tokens": 7675416.0,
+      "reward": 0.7490234375,
+      "reward_std": 0.009653441607952118,
+      "rewards//mean": 0.7490234375,
+      "rewards//std": 0.04129883274435997,
+      "step": 888
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1778,
+      "grad_norm": 0.5779797434806824,
+      "kl": 0.29215819016098976,
+      "learning_rate": 4.654676626291123e-06,
+      "loss": 0.0117,
+      "num_tokens": 7684048.0,
+      "reward": 0.75439453125,
+      "reward_std": 0.010933919809758663,
+      "rewards//mean": 0.75439453125,
+      "rewards//std": 0.03273756802082062,
+      "step": 889
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.178,
+      "grad_norm": 0.8437680602073669,
+      "kl": 0.2626949865370989,
+      "learning_rate": 4.653871550910414e-06,
+      "loss": 0.0105,
+      "num_tokens": 7692680.0,
+      "reward": 0.7613525390625,
+      "reward_std": 0.010730916634202003,
+      "rewards//mean": 0.7613525390625,
+      "rewards//std": 0.02644515223801136,
+      "step": 890
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1782,
+      "grad_norm": 0.5432496666908264,
+      "kl": 0.2565987464040518,
+      "learning_rate": 4.653065607950502e-06,
+      "loss": 0.0103,
+      "num_tokens": 7701288.0,
+      "reward": 0.7493896484375,
+      "reward_std": 0.013223410584032536,
+      "rewards//mean": 0.7493896484375,
+      "rewards//std": 0.03801432624459267,
+      "step": 891
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1784,
+      "grad_norm": 0.6957722902297974,
+      "kl": 0.25557348132133484,
+      "learning_rate": 4.65225879773602e-06,
+      "loss": 0.0102,
+      "num_tokens": 7709928.0,
+      "reward": 0.76568603515625,
+      "reward_std": 0.008938233368098736,
+      "rewards//mean": 0.76568603515625,
+      "rewards//std": 0.023193299770355225,
+      "step": 892
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1786,
+      "grad_norm": 0.526515781879425,
+      "kl": 0.2758950814604759,
+      "learning_rate": 4.651451120591952e-06,
+      "loss": 0.011,
+      "num_tokens": 7718504.0,
+      "reward": 0.760009765625,
+      "reward_std": 0.008625369518995285,
+      "rewards//mean": 0.760009765625,
+      "rewards//std": 0.0195234976708889,
+      "step": 893
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1788,
+      "grad_norm": 0.6605725884437561,
+      "kl": 0.270997678861022,
+      "learning_rate": 4.650642576843631e-06,
+      "loss": 0.0108,
+      "num_tokens": 7727272.0,
+      "reward": 0.76007080078125,
+      "reward_std": 0.01003330573439598,
+      "rewards//mean": 0.76007080078125,
+      "rewards//std": 0.03080352023243904,
+      "step": 894
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.179,
+      "grad_norm": 0.5448390245437622,
+      "kl": 0.252898957580328,
+      "learning_rate": 4.649833166816736e-06,
+      "loss": 0.0101,
+      "num_tokens": 7736008.0,
+      "reward": 0.73876953125,
+      "reward_std": 0.011466844007372856,
+      "rewards//mean": 0.73876953125,
+      "rewards//std": 0.03744480386376381,
+      "step": 895
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1792,
+      "grad_norm": 0.6046472191810608,
+      "kl": 0.2753715682774782,
+      "learning_rate": 4.649022890837298e-06,
+      "loss": 0.011,
+      "num_tokens": 7744680.0,
+      "reward": 0.749267578125,
+      "reward_std": 0.009591508656740189,
+      "rewards//mean": 0.749267578125,
+      "rewards//std": 0.02788209356367588,
+      "step": 896
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1794,
+      "grad_norm": 0.8163694739341736,
+      "kl": 0.2629574202001095,
+      "learning_rate": 4.648211749231698e-06,
+      "loss": 0.0105,
+      "num_tokens": 7753328.0,
+      "reward": 0.78839111328125,
+      "reward_std": 0.00975382886826992,
+      "rewards//mean": 0.78839111328125,
+      "rewards//std": 0.02201532945036888,
+      "step": 897
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1796,
+      "grad_norm": 0.8590662479400635,
+      "kl": 0.2539634648710489,
+      "learning_rate": 4.6473997423266615e-06,
+      "loss": 0.0102,
+      "num_tokens": 7761912.0,
+      "reward": 0.765380859375,
+      "reward_std": 0.007768705021589994,
+      "rewards//mean": 0.765380859375,
+      "rewards//std": 0.016126545146107674,
+      "step": 898
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1798,
+      "grad_norm": 0.5974591970443726,
+      "kl": 0.26154331117868423,
+      "learning_rate": 4.646586870449266e-06,
+      "loss": 0.0105,
+      "num_tokens": 7770584.0,
+      "reward": 0.761962890625,
+      "reward_std": 0.013042537495493889,
+      "rewards//mean": 0.761962890625,
+      "rewards//std": 0.034868303686380386,
+      "step": 899
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.18,
+      "grad_norm": 0.8781308531761169,
+      "kl": 0.2626851834356785,
+      "learning_rate": 4.645773133926936e-06,
+      "loss": 0.0105,
+      "num_tokens": 7779176.0,
+      "reward": 0.75445556640625,
+      "reward_std": 0.010092251002788544,
+      "rewards//mean": 0.75445556640625,
+      "rewards//std": 0.03166474774479866,
+      "step": 900
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1802,
+      "grad_norm": 0.6971238255500793,
+      "kl": 0.25614192336797714,
+      "learning_rate": 4.644958533087443e-06,
+      "loss": 0.0102,
+      "num_tokens": 7787928.0,
+      "reward": 0.75091552734375,
+      "reward_std": 0.008847690187394619,
+      "rewards//mean": 0.75091552734375,
+      "rewards//std": 0.024505889043211937,
+      "step": 901
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1804,
+      "grad_norm": 0.6750687956809998,
+      "kl": 0.26914981193840504,
+      "learning_rate": 4.64414306825891e-06,
+      "loss": 0.0108,
+      "num_tokens": 7796560.0,
+      "reward": 0.7652587890625,
+      "reward_std": 0.012014053761959076,
+      "rewards//mean": 0.7652587890625,
+      "rewards//std": 0.035356976091861725,
+      "step": 902
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1806,
+      "grad_norm": 0.6785114407539368,
+      "kl": 0.2674534786492586,
+      "learning_rate": 4.643326739769805e-06,
+      "loss": 0.0107,
+      "num_tokens": 7805264.0,
+      "reward": 0.72943115234375,
+      "reward_std": 0.011744974181056023,
+      "rewards//mean": 0.72943115234375,
+      "rewards//std": 0.037007205188274384,
+      "step": 903
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1808,
+      "grad_norm": 0.6502825617790222,
+      "kl": 0.2490905374288559,
+      "learning_rate": 4.642509547948947e-06,
+      "loss": 0.01,
+      "num_tokens": 7813920.0,
+      "reward": 0.7679443359375,
+      "reward_std": 0.009585111401975155,
+      "rewards//mean": 0.7679443359375,
+      "rewards//std": 0.020946016535162926,
+      "step": 904
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.181,
+      "grad_norm": 0.8235167860984802,
+      "kl": 0.27164326049387455,
+      "learning_rate": 4.6416914931254984e-06,
+      "loss": 0.0109,
+      "num_tokens": 7822416.0,
+      "reward": 0.70794677734375,
+      "reward_std": 0.010866380296647549,
+      "rewards//mean": 0.70794677734375,
+      "rewards//std": 0.040611300617456436,
+      "step": 905
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1812,
+      "grad_norm": 1.0692238807678223,
+      "kl": 0.28652898594737053,
+      "learning_rate": 4.640872575628973e-06,
+      "loss": 0.0115,
+      "num_tokens": 7831000.0,
+      "reward": 0.728271484375,
+      "reward_std": 0.01055578701198101,
+      "rewards//mean": 0.728271484375,
+      "rewards//std": 0.03541964292526245,
+      "step": 906
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1814,
+      "grad_norm": 0.9947613477706909,
+      "kl": 0.2775047402828932,
+      "learning_rate": 4.6400527957892295e-06,
+      "loss": 0.0111,
+      "num_tokens": 7839616.0,
+      "reward": 0.76861572265625,
+      "reward_std": 0.01045004278421402,
+      "rewards//mean": 0.76861572265625,
+      "rewards//std": 0.025683382526040077,
+      "step": 907
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1816,
+      "grad_norm": 0.7290713787078857,
+      "kl": 0.27869971096515656,
+      "learning_rate": 4.639232153936476e-06,
+      "loss": 0.0111,
+      "num_tokens": 7848272.0,
+      "reward": 0.78680419921875,
+      "reward_std": 0.007229713257402182,
+      "rewards//mean": 0.78680419921875,
+      "rewards//std": 0.03553522005677223,
+      "step": 908
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1818,
+      "grad_norm": 0.7138441801071167,
+      "kl": 0.27684359066188335,
+      "learning_rate": 4.638410650401267e-06,
+      "loss": 0.0111,
+      "num_tokens": 7856824.0,
+      "reward": 0.75946044921875,
+      "reward_std": 0.009466202929615974,
+      "rewards//mean": 0.75946044921875,
+      "rewards//std": 0.018594926223158836,
+      "step": 909
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.182,
+      "grad_norm": 0.7739296555519104,
+      "kl": 0.2615481149405241,
+      "learning_rate": 4.637588285514504e-06,
+      "loss": 0.0105,
+      "num_tokens": 7865384.0,
+      "reward": 0.7427978515625,
+      "reward_std": 0.008604985661804676,
+      "rewards//mean": 0.7427978515625,
+      "rewards//std": 0.02929145097732544,
+      "step": 910
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1822,
+      "grad_norm": 0.9076600670814514,
+      "kl": 0.2758956626057625,
+      "learning_rate": 4.636765059607434e-06,
+      "loss": 0.011,
+      "num_tokens": 7874016.0,
+      "reward": 0.765380859375,
+      "reward_std": 0.008405257016420364,
+      "rewards//mean": 0.765380859375,
+      "rewards//std": 0.02437838539481163,
+      "step": 911
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1824,
+      "grad_norm": 0.7445156574249268,
+      "kl": 0.2667129561305046,
+      "learning_rate": 4.6359409730116546e-06,
+      "loss": 0.0107,
+      "num_tokens": 7882688.0,
+      "reward": 0.78369140625,
+      "reward_std": 0.0080318758264184,
+      "rewards//mean": 0.78369140625,
+      "rewards//std": 0.023983998224139214,
+      "step": 912
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1826,
+      "grad_norm": 0.8461691737174988,
+      "kl": 0.2566282209008932,
+      "learning_rate": 4.635116026059107e-06,
+      "loss": 0.0103,
+      "num_tokens": 7891408.0,
+      "reward": 0.76092529296875,
+      "reward_std": 0.006388828158378601,
+      "rewards//mean": 0.76092529296875,
+      "rewards//std": 0.024912068620324135,
+      "step": 913
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1828,
+      "grad_norm": 1.0944255590438843,
+      "kl": 0.27355407923460007,
+      "learning_rate": 4.634290219082078e-06,
+      "loss": 0.0109,
+      "num_tokens": 7900024.0,
+      "reward": 0.7568359375,
+      "reward_std": 0.00810672901570797,
+      "rewards//mean": 0.7568359375,
+      "rewards//std": 0.029487434774637222,
+      "step": 914
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.183,
+      "grad_norm": 1.0639708042144775,
+      "kl": 0.28650105744600296,
+      "learning_rate": 4.633463552413205e-06,
+      "loss": 0.0115,
+      "num_tokens": 7908696.0,
+      "reward": 0.76947021484375,
+      "reward_std": 0.006652969866991043,
+      "rewards//mean": 0.76947021484375,
+      "rewards//std": 0.024434128776192665,
+      "step": 915
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1832,
+      "grad_norm": 0.8133704662322998,
+      "kl": 0.2669271398335695,
+      "learning_rate": 4.632636026385468e-06,
+      "loss": 0.0107,
+      "num_tokens": 7917392.0,
+      "reward": 0.7740478515625,
+      "reward_std": 0.011425754986703396,
+      "rewards//mean": 0.7740478515625,
+      "rewards//std": 0.01731915958225727,
+      "step": 916
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1834,
+      "grad_norm": 1.5723562240600586,
+      "kl": 0.27988065406680107,
+      "learning_rate": 4.631807641332195e-06,
+      "loss": 0.0112,
+      "num_tokens": 7926048.0,
+      "reward": 0.734375,
+      "reward_std": 0.005926807411015034,
+      "rewards//mean": 0.734375,
+      "rewards//std": 0.04288693889975548,
+      "step": 917
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1836,
+      "grad_norm": 1.0416011810302734,
+      "kl": 0.3021409697830677,
+      "learning_rate": 4.630978397587058e-06,
+      "loss": 0.0121,
+      "num_tokens": 7935016.0,
+      "reward": 0.77471923828125,
+      "reward_std": 0.00614708149805665,
+      "rewards//mean": 0.77471923828125,
+      "rewards//std": 0.040547508746385574,
+      "step": 918
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1838,
+      "grad_norm": 0.7368637919425964,
+      "kl": 0.2729188334196806,
+      "learning_rate": 4.630148295484078e-06,
+      "loss": 0.0109,
+      "num_tokens": 7943656.0,
+      "reward": 0.74053955078125,
+      "reward_std": 0.007835205644369125,
+      "rewards//mean": 0.74053955078125,
+      "rewards//std": 0.025862522423267365,
+      "step": 919
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.184,
+      "grad_norm": 1.2126497030258179,
+      "kl": 0.279952809214592,
+      "learning_rate": 4.62931733535762e-06,
+      "loss": 0.0112,
+      "num_tokens": 7952288.0,
+      "reward": 0.73394775390625,
+      "reward_std": 0.007915819063782692,
+      "rewards//mean": 0.73394775390625,
+      "rewards//std": 0.030266623944044113,
+      "step": 920
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1842,
+      "grad_norm": 1.5177454948425293,
+      "kl": 0.28335076197981834,
+      "learning_rate": 4.628485517542393e-06,
+      "loss": 0.0113,
+      "num_tokens": 7960904.0,
+      "reward": 0.74371337890625,
+      "reward_std": 0.008045843802392483,
+      "rewards//mean": 0.74371337890625,
+      "rewards//std": 0.03372957557439804,
+      "step": 921
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1844,
+      "grad_norm": 0.8178320527076721,
+      "kl": 0.2918772976845503,
+      "learning_rate": 4.627652842373454e-06,
+      "loss": 0.0117,
+      "num_tokens": 7969416.0,
+      "reward": 0.77789306640625,
+      "reward_std": 0.009983016178011894,
+      "rewards//mean": 0.77789306640625,
+      "rewards//std": 0.024058280512690544,
+      "step": 922
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1846,
+      "grad_norm": 1.0449796915054321,
+      "kl": 0.3009101003408432,
+      "learning_rate": 4.626819310186204e-06,
+      "loss": 0.012,
+      "num_tokens": 7978048.0,
+      "reward": 0.76824951171875,
+      "reward_std": 0.009645262733101845,
+      "rewards//mean": 0.76824951171875,
+      "rewards//std": 0.027419421821832657,
+      "step": 923
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1848,
+      "grad_norm": 0.9068644642829895,
+      "kl": 0.2759493812918663,
+      "learning_rate": 4.625984921316392e-06,
+      "loss": 0.011,
+      "num_tokens": 7986648.0,
+      "reward": 0.76788330078125,
+      "reward_std": 0.011120768263936043,
+      "rewards//mean": 0.76788330078125,
+      "rewards//std": 0.02279633842408657,
+      "step": 924
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.185,
+      "grad_norm": 1.1586679220199585,
+      "kl": 0.2916589751839638,
+      "learning_rate": 4.625149676100107e-06,
+      "loss": 0.0117,
+      "num_tokens": 7995320.0,
+      "reward": 0.76434326171875,
+      "reward_std": 0.0063139949925243855,
+      "rewards//mean": 0.76434326171875,
+      "rewards//std": 0.022053800523281097,
+      "step": 925
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1852,
+      "grad_norm": 0.9030797481536865,
+      "kl": 0.28806450217962265,
+      "learning_rate": 4.624313574873787e-06,
+      "loss": 0.0115,
+      "num_tokens": 8003952.0,
+      "reward": 0.7706298828125,
+      "reward_std": 0.010051582008600235,
+      "rewards//mean": 0.7706298828125,
+      "rewards//std": 0.03246990218758583,
+      "step": 926
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1854,
+      "grad_norm": 0.8641966581344604,
+      "kl": 0.29965170845389366,
+      "learning_rate": 4.623476617974212e-06,
+      "loss": 0.012,
+      "num_tokens": 8012560.0,
+      "reward": 0.76129150390625,
+      "reward_std": 0.006479769945144653,
+      "rewards//mean": 0.76129150390625,
+      "rewards//std": 0.02804011106491089,
+      "step": 927
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1856,
+      "grad_norm": 0.9614176154136658,
+      "kl": 0.30398988723754883,
+      "learning_rate": 4.62263880573851e-06,
+      "loss": 0.0122,
+      "num_tokens": 8021216.0,
+      "reward": 0.77435302734375,
+      "reward_std": 0.00971104484051466,
+      "rewards//mean": 0.77435302734375,
+      "rewards//std": 0.02430741675198078,
+      "step": 928
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1858,
+      "grad_norm": 0.7756906747817993,
+      "kl": 0.30807607248425484,
+      "learning_rate": 4.6218001385041504e-06,
+      "loss": 0.0123,
+      "num_tokens": 8029968.0,
+      "reward": 0.74884033203125,
+      "reward_std": 0.012052865698933601,
+      "rewards//mean": 0.74884033203125,
+      "rewards//std": 0.03312136232852936,
+      "step": 929
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.186,
+      "grad_norm": 1.1806690692901611,
+      "kl": 0.2854270339012146,
+      "learning_rate": 4.6209606166089495e-06,
+      "loss": 0.0114,
+      "num_tokens": 8038632.0,
+      "reward": 0.75872802734375,
+      "reward_std": 0.006012373138219118,
+      "rewards//mean": 0.75872802734375,
+      "rewards//std": 0.02449600212275982,
+      "step": 930
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1862,
+      "grad_norm": 0.8380911350250244,
+      "kl": 0.2782778702676296,
+      "learning_rate": 4.620120240391065e-06,
+      "loss": 0.0111,
+      "num_tokens": 8047248.0,
+      "reward": 0.74365234375,
+      "reward_std": 0.009586180560290813,
+      "rewards//mean": 0.74365234375,
+      "rewards//std": 0.023172486573457718,
+      "step": 931
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1864,
+      "grad_norm": 1.0672872066497803,
+      "kl": 0.29075466096401215,
+      "learning_rate": 4.619279010189002e-06,
+      "loss": 0.0116,
+      "num_tokens": 8055896.0,
+      "reward": 0.737060546875,
+      "reward_std": 0.008078483864665031,
+      "rewards//mean": 0.737060546875,
+      "rewards//std": 0.03260691091418266,
+      "step": 932
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1866,
+      "grad_norm": 0.937512993812561,
+      "kl": 0.2672659792006016,
+      "learning_rate": 4.618436926341607e-06,
+      "loss": 0.0107,
+      "num_tokens": 8064552.0,
+      "reward": 0.77001953125,
+      "reward_std": 0.01028747484087944,
+      "rewards//mean": 0.77001953125,
+      "rewards//std": 0.021871235221624374,
+      "step": 933
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1868,
+      "grad_norm": 0.7125497460365295,
+      "kl": 0.2997877672314644,
+      "learning_rate": 4.617593989188071e-06,
+      "loss": 0.012,
+      "num_tokens": 8073232.0,
+      "reward": 0.7581787109375,
+      "reward_std": 0.009922824800014496,
+      "rewards//mean": 0.7581787109375,
+      "rewards//std": 0.03353586792945862,
+      "step": 934
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.187,
+      "grad_norm": 0.9831026196479797,
+      "kl": 0.30767880380153656,
+      "learning_rate": 4.616750199067929e-06,
+      "loss": 0.0123,
+      "num_tokens": 8081840.0,
+      "reward": 0.7542724609375,
+      "reward_std": 0.012375378049910069,
+      "rewards//mean": 0.7542724609375,
+      "rewards//std": 0.03080284409224987,
+      "step": 935
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1872,
+      "grad_norm": 1.1639822721481323,
+      "kl": 0.3031549211591482,
+      "learning_rate": 4.615905556321061e-06,
+      "loss": 0.0121,
+      "num_tokens": 8090544.0,
+      "reward": 0.75262451171875,
+      "reward_std": 0.01033235713839531,
+      "rewards//mean": 0.75262451171875,
+      "rewards//std": 0.027613069862127304,
+      "step": 936
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1874,
+      "grad_norm": 0.8558388352394104,
+      "kl": 0.30340132489800453,
+      "learning_rate": 4.615060061287688e-06,
+      "loss": 0.0121,
+      "num_tokens": 8099208.0,
+      "reward": 0.7528076171875,
+      "reward_std": 0.013534157536923885,
+      "rewards//mean": 0.7528076171875,
+      "rewards//std": 0.027782833203673363,
+      "step": 937
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1876,
+      "grad_norm": 0.845984160900116,
+      "kl": 0.29561108350753784,
+      "learning_rate": 4.614213714308374e-06,
+      "loss": 0.0118,
+      "num_tokens": 8107856.0,
+      "reward": 0.75341796875,
+      "reward_std": 0.011012168601155281,
+      "rewards//mean": 0.75341796875,
+      "rewards//std": 0.02374039590358734,
+      "step": 938
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1878,
+      "grad_norm": 0.8777411580085754,
+      "kl": 0.29283004254102707,
+      "learning_rate": 4.6133665157240306e-06,
+      "loss": 0.0117,
+      "num_tokens": 8116456.0,
+      "reward": 0.71600341796875,
+      "reward_std": 0.008553056046366692,
+      "rewards//mean": 0.71600341796875,
+      "rewards//std": 0.0435798205435276,
+      "step": 939
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.188,
+      "grad_norm": 0.9265841841697693,
+      "kl": 0.29388684406876564,
+      "learning_rate": 4.612518465875906e-06,
+      "loss": 0.0118,
+      "num_tokens": 8125016.0,
+      "reward": 0.7520751953125,
+      "reward_std": 0.009404158219695091,
+      "rewards//mean": 0.7520751953125,
+      "rewards//std": 0.03676244989037514,
+      "step": 940
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1882,
+      "grad_norm": 0.8138207793235779,
+      "kl": 0.3315567076206207,
+      "learning_rate": 4.611669565105597e-06,
+      "loss": 0.0133,
+      "num_tokens": 8133600.0,
+      "reward": 0.7581787109375,
+      "reward_std": 0.007814617827534676,
+      "rewards//mean": 0.7581787109375,
+      "rewards//std": 0.029054835438728333,
+      "step": 941
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1884,
+      "grad_norm": 0.895754337310791,
+      "kl": 0.2886655665934086,
+      "learning_rate": 4.610819813755038e-06,
+      "loss": 0.0115,
+      "num_tokens": 8142232.0,
+      "reward": 0.7618408203125,
+      "reward_std": 0.008037852123379707,
+      "rewards//mean": 0.7618408203125,
+      "rewards//std": 0.018055196851491928,
+      "step": 942
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1886,
+      "grad_norm": 1.0638179779052734,
+      "kl": 0.33327390998601913,
+      "learning_rate": 4.609969212166512e-06,
+      "loss": 0.0133,
+      "num_tokens": 8150976.0,
+      "reward": 0.7430419921875,
+      "reward_std": 0.008485731668770313,
+      "rewards//mean": 0.7430419921875,
+      "rewards//std": 0.023328416049480438,
+      "step": 943
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1888,
+      "grad_norm": 1.1232788562774658,
+      "kl": 0.31669921800494194,
+      "learning_rate": 4.609117760682639e-06,
+      "loss": 0.0127,
+      "num_tokens": 8159640.0,
+      "reward": 0.7677001953125,
+      "reward_std": 0.010280384682118893,
+      "rewards//mean": 0.7677001953125,
+      "rewards//std": 0.039260219782590866,
+      "step": 944
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.189,
+      "grad_norm": 1.87589430809021,
+      "kl": 0.2973406966775656,
+      "learning_rate": 4.608265459646384e-06,
+      "loss": 0.0119,
+      "num_tokens": 8168288.0,
+      "reward": 0.73162841796875,
+      "reward_std": 0.008025545626878738,
+      "rewards//mean": 0.73162841796875,
+      "rewards//std": 0.028157014399766922,
+      "step": 945
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1892,
+      "grad_norm": 0.7903565168380737,
+      "kl": 0.2874004878103733,
+      "learning_rate": 4.607412309401054e-06,
+      "loss": 0.0115,
+      "num_tokens": 8176928.0,
+      "reward": 0.77008056640625,
+      "reward_std": 0.006889053154736757,
+      "rewards//mean": 0.77008056640625,
+      "rewards//std": 0.02101699262857437,
+      "step": 946
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1894,
+      "grad_norm": 1.0100493431091309,
+      "kl": 0.28360038809478283,
+      "learning_rate": 4.606558310290298e-06,
+      "loss": 0.0113,
+      "num_tokens": 8185472.0,
+      "reward": 0.7728271484375,
+      "reward_std": 0.007974790409207344,
+      "rewards//mean": 0.7728271484375,
+      "rewards//std": 0.021634282544255257,
+      "step": 947
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1896,
+      "grad_norm": 0.9846349358558655,
+      "kl": 0.3387889303267002,
+      "learning_rate": 4.605703462658107e-06,
+      "loss": 0.0136,
+      "num_tokens": 8194096.0,
+      "reward": 0.77178955078125,
+      "reward_std": 0.01358504593372345,
+      "rewards//mean": 0.77178955078125,
+      "rewards//std": 0.02849157713353634,
+      "step": 948
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1898,
+      "grad_norm": 1.747856616973877,
+      "kl": 0.33013427443802357,
+      "learning_rate": 4.604847766848812e-06,
+      "loss": 0.0132,
+      "num_tokens": 8202632.0,
+      "reward": 0.7506103515625,
+      "reward_std": 0.011012842878699303,
+      "rewards//mean": 0.7506103515625,
+      "rewards//std": 0.0204012431204319,
+      "step": 949
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.19,
+      "grad_norm": 0.925475537776947,
+      "kl": 0.31422964110970497,
+      "learning_rate": 4.60399122320709e-06,
+      "loss": 0.0126,
+      "num_tokens": 8211200.0,
+      "reward": 0.76910400390625,
+      "reward_std": 0.009690100327134132,
+      "rewards//mean": 0.76910400390625,
+      "rewards//std": 0.018468312919139862,
+      "step": 950
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1902,
+      "grad_norm": 1.132817268371582,
+      "kl": 0.3214902691543102,
+      "learning_rate": 4.603133832077953e-06,
+      "loss": 0.0129,
+      "num_tokens": 8219872.0,
+      "reward": 0.7730712890625,
+      "reward_std": 0.01109931617975235,
+      "rewards//mean": 0.7730712890625,
+      "rewards//std": 0.030335379764437675,
+      "step": 951
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1904,
+      "grad_norm": 0.914539635181427,
+      "kl": 0.3235311582684517,
+      "learning_rate": 4.602275593806761e-06,
+      "loss": 0.0129,
+      "num_tokens": 8228504.0,
+      "reward": 0.76373291015625,
+      "reward_std": 0.008837835863232613,
+      "rewards//mean": 0.76373291015625,
+      "rewards//std": 0.01813581772148609,
+      "step": 952
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1906,
+      "grad_norm": 0.9955701231956482,
+      "kl": 0.34584128484129906,
+      "learning_rate": 4.601416508739211e-06,
+      "loss": 0.0138,
+      "num_tokens": 8237192.0,
+      "reward": 0.74029541015625,
+      "reward_std": 0.012629736214876175,
+      "rewards//mean": 0.74029541015625,
+      "rewards//std": 0.027743780985474586,
+      "step": 953
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1908,
+      "grad_norm": 0.9932671785354614,
+      "kl": 0.31934236362576485,
+      "learning_rate": 4.600556577221342e-06,
+      "loss": 0.0128,
+      "num_tokens": 8245880.0,
+      "reward": 0.73089599609375,
+      "reward_std": 0.008857986889779568,
+      "rewards//mean": 0.73089599609375,
+      "rewards//std": 0.031467169523239136,
+      "step": 954
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.191,
+      "grad_norm": 1.2234418392181396,
+      "kl": 0.30861231684684753,
+      "learning_rate": 4.599695799599537e-06,
+      "loss": 0.0123,
+      "num_tokens": 8254472.0,
+      "reward": 0.78582763671875,
+      "reward_std": 0.00961345061659813,
+      "rewards//mean": 0.78582763671875,
+      "rewards//std": 0.023318924009799957,
+      "step": 955
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1912,
+      "grad_norm": 1.053141474723816,
+      "kl": 0.3571406714618206,
+      "learning_rate": 4.5988341762205125e-06,
+      "loss": 0.0143,
+      "num_tokens": 8263168.0,
+      "reward": 0.76080322265625,
+      "reward_std": 0.008980470709502697,
+      "rewards//mean": 0.76080322265625,
+      "rewards//std": 0.023195909336209297,
+      "step": 956
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1914,
+      "grad_norm": 1.2716294527053833,
+      "kl": 0.3623424470424652,
+      "learning_rate": 4.5979717074313336e-06,
+      "loss": 0.0145,
+      "num_tokens": 8271784.0,
+      "reward": 0.7208251953125,
+      "reward_std": 0.011484457179903984,
+      "rewards//mean": 0.7208251953125,
+      "rewards//std": 0.04424877464771271,
+      "step": 957
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1916,
+      "grad_norm": 1.2183886766433716,
+      "kl": 0.3608044385910034,
+      "learning_rate": 4.5971083935794026e-06,
+      "loss": 0.0144,
+      "num_tokens": 8280496.0,
+      "reward": 0.74468994140625,
+      "reward_std": 0.00796246062964201,
+      "rewards//mean": 0.74468994140625,
+      "rewards//std": 0.03260313719511032,
+      "step": 958
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1918,
+      "grad_norm": 1.0903003215789795,
+      "kl": 0.3438631668686867,
+      "learning_rate": 4.5962442350124605e-06,
+      "loss": 0.0138,
+      "num_tokens": 8289160.0,
+      "reward": 0.75726318359375,
+      "reward_std": 0.008945617824792862,
+      "rewards//mean": 0.75726318359375,
+      "rewards//std": 0.02822735533118248,
+      "step": 959
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.192,
+      "grad_norm": 1.063489556312561,
+      "kl": 0.3416360281407833,
+      "learning_rate": 4.595379232078592e-06,
+      "loss": 0.0137,
+      "num_tokens": 8297768.0,
+      "reward": 0.72540283203125,
+      "reward_std": 0.010138707235455513,
+      "rewards//mean": 0.72540283203125,
+      "rewards//std": 0.03077893704175949,
+      "step": 960
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1922,
+      "grad_norm": 0.9618769884109497,
+      "kl": 0.3541461080312729,
+      "learning_rate": 4.5945133851262185e-06,
+      "loss": 0.0142,
+      "num_tokens": 8306440.0,
+      "reward": 0.7489013671875,
+      "reward_std": 0.008929513394832611,
+      "rewards//mean": 0.7489013671875,
+      "rewards//std": 0.029630571603775024,
+      "step": 961
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1924,
+      "grad_norm": 1.2882936000823975,
+      "kl": 0.36649367958307266,
+      "learning_rate": 4.593646694504105e-06,
+      "loss": 0.0147,
+      "num_tokens": 8315072.0,
+      "reward": 0.75372314453125,
+      "reward_std": 0.010234087705612183,
+      "rewards//mean": 0.75372314453125,
+      "rewards//std": 0.025395916774868965,
+      "step": 962
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1926,
+      "grad_norm": 1.241235613822937,
+      "kl": 0.3214772678911686,
+      "learning_rate": 4.5927791605613525e-06,
+      "loss": 0.0129,
+      "num_tokens": 8323776.0,
+      "reward": 0.76800537109375,
+      "reward_std": 0.008909153752028942,
+      "rewards//mean": 0.76800537109375,
+      "rewards//std": 0.02892867475748062,
+      "step": 963
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1928,
+      "grad_norm": 1.4344669580459595,
+      "kl": 0.36224448308348656,
+      "learning_rate": 4.591910783647405e-06,
+      "loss": 0.0145,
+      "num_tokens": 8332408.0,
+      "reward": 0.75018310546875,
+      "reward_std": 0.011086471378803253,
+      "rewards//mean": 0.75018310546875,
+      "rewards//std": 0.029143985360860825,
+      "step": 964
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.193,
+      "grad_norm": 1.057570457458496,
+      "kl": 0.3504234068095684,
+      "learning_rate": 4.591041564112043e-06,
+      "loss": 0.014,
+      "num_tokens": 8341096.0,
+      "reward": 0.75848388671875,
+      "reward_std": 0.011695520021021366,
+      "rewards//mean": 0.75848388671875,
+      "rewards//std": 0.023246105760335922,
+      "step": 965
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1932,
+      "grad_norm": 1.2750986814498901,
+      "kl": 0.3357960321009159,
+      "learning_rate": 4.59017150230539e-06,
+      "loss": 0.0134,
+      "num_tokens": 8349760.0,
+      "reward": 0.7686767578125,
+      "reward_std": 0.011024602688848972,
+      "rewards//mean": 0.7686767578125,
+      "rewards//std": 0.02808629721403122,
+      "step": 966
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1934,
+      "grad_norm": 1.308644413948059,
+      "kl": 0.35375194624066353,
+      "learning_rate": 4.589300598577906e-06,
+      "loss": 0.0142,
+      "num_tokens": 8358440.0,
+      "reward": 0.75726318359375,
+      "reward_std": 0.0074037788435816765,
+      "rewards//mean": 0.75726318359375,
+      "rewards//std": 0.027497153729200363,
+      "step": 967
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1936,
+      "grad_norm": 1.059302568435669,
+      "kl": 0.3852734975516796,
+      "learning_rate": 4.58842885328039e-06,
+      "loss": 0.0154,
+      "num_tokens": 8367064.0,
+      "reward": 0.745361328125,
+      "reward_std": 0.006511340849101543,
+      "rewards//mean": 0.745361328125,
+      "rewards//std": 0.03340674936771393,
+      "step": 968
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1938,
+      "grad_norm": 1.2672991752624512,
+      "kl": 0.3788112476468086,
+      "learning_rate": 4.587556266763982e-06,
+      "loss": 0.0152,
+      "num_tokens": 8375712.0,
+      "reward": 0.72161865234375,
+      "reward_std": 0.0076048411428928375,
+      "rewards//mean": 0.72161865234375,
+      "rewards//std": 0.02709229476749897,
+      "step": 969
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.194,
+      "grad_norm": 1.2441980838775635,
+      "kl": 0.34678515046834946,
+      "learning_rate": 4.586682839380159e-06,
+      "loss": 0.0139,
+      "num_tokens": 8384464.0,
+      "reward": 0.75909423828125,
+      "reward_std": 0.011957092210650444,
+      "rewards//mean": 0.75909423828125,
+      "rewards//std": 0.029256997630000114,
+      "step": 970
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1942,
+      "grad_norm": 1.481521725654602,
+      "kl": 0.3327783904969692,
+      "learning_rate": 4.585808571480739e-06,
+      "loss": 0.0133,
+      "num_tokens": 8393096.0,
+      "reward": 0.76458740234375,
+      "reward_std": 0.009498574770987034,
+      "rewards//mean": 0.76458740234375,
+      "rewards//std": 0.02750045619904995,
+      "step": 971
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1944,
+      "grad_norm": 1.0270497798919678,
+      "kl": 0.35905660316348076,
+      "learning_rate": 4.584933463417874e-06,
+      "loss": 0.0144,
+      "num_tokens": 8401712.0,
+      "reward": 0.7547607421875,
+      "reward_std": 0.009442451409995556,
+      "rewards//mean": 0.7547607421875,
+      "rewards//std": 0.02266489528119564,
+      "step": 972
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1946,
+      "grad_norm": 1.9370031356811523,
+      "kl": 0.4017368145287037,
+      "learning_rate": 4.584057515544061e-06,
+      "loss": 0.0161,
+      "num_tokens": 8410496.0,
+      "reward": 0.75067138671875,
+      "reward_std": 0.008788703009486198,
+      "rewards//mean": 0.75067138671875,
+      "rewards//std": 0.026213670149445534,
+      "step": 973
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1948,
+      "grad_norm": 2.0624356269836426,
+      "kl": 0.36438558250665665,
+      "learning_rate": 4.583180728212128e-06,
+      "loss": 0.0146,
+      "num_tokens": 8419064.0,
+      "reward": 0.75830078125,
+      "reward_std": 0.00762117188423872,
+      "rewards//mean": 0.75830078125,
+      "rewards//std": 0.02847197651863098,
+      "step": 974
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.195,
+      "grad_norm": 1.3694345951080322,
+      "kl": 0.3885602429509163,
+      "learning_rate": 4.582303101775249e-06,
+      "loss": 0.0155,
+      "num_tokens": 8427840.0,
+      "reward": 0.76422119140625,
+      "reward_std": 0.009295577183365822,
+      "rewards//mean": 0.76422119140625,
+      "rewards//std": 0.02592390775680542,
+      "step": 975
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1952,
+      "grad_norm": 1.3455674648284912,
+      "kl": 0.3587591424584389,
+      "learning_rate": 4.5814246365869285e-06,
+      "loss": 0.0144,
+      "num_tokens": 8436616.0,
+      "reward": 0.780029296875,
+      "reward_std": 0.00841439887881279,
+      "rewards//mean": 0.780029296875,
+      "rewards//std": 0.022754548117518425,
+      "step": 976
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1954,
+      "grad_norm": 1.3943334817886353,
+      "kl": 0.3845372460782528,
+      "learning_rate": 4.580545333001014e-06,
+      "loss": 0.0154,
+      "num_tokens": 8445216.0,
+      "reward": 0.7724609375,
+      "reward_std": 0.010662311688065529,
+      "rewards//mean": 0.7724609375,
+      "rewards//std": 0.03599030151963234,
+      "step": 977
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1956,
+      "grad_norm": 1.7175418138504028,
+      "kl": 0.3837543651461601,
+      "learning_rate": 4.579665191371687e-06,
+      "loss": 0.0154,
+      "num_tokens": 8453896.0,
+      "reward": 0.777099609375,
+      "reward_std": 0.0080185541883111,
+      "rewards//mean": 0.777099609375,
+      "rewards//std": 0.027151526883244514,
+      "step": 978
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1958,
+      "grad_norm": 1.7660549879074097,
+      "kl": 0.4039181172847748,
+      "learning_rate": 4.578784212053471e-06,
+      "loss": 0.0162,
+      "num_tokens": 8462552.0,
+      "reward": 0.7745361328125,
+      "reward_std": 0.0088932104408741,
+      "rewards//mean": 0.7745361328125,
+      "rewards//std": 0.018208811059594154,
+      "step": 979
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.196,
+      "grad_norm": 1.3596855401992798,
+      "kl": 0.41640326753258705,
+      "learning_rate": 4.577902395401222e-06,
+      "loss": 0.0167,
+      "num_tokens": 8471232.0,
+      "reward": 0.749755859375,
+      "reward_std": 0.012476136907935143,
+      "rewards//mean": 0.749755859375,
+      "rewards//std": 0.0270084235817194,
+      "step": 980
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1962,
+      "grad_norm": 1.7318655252456665,
+      "kl": 0.3891097605228424,
+      "learning_rate": 4.577019741770137e-06,
+      "loss": 0.0156,
+      "num_tokens": 8479856.0,
+      "reward": 0.763427734375,
+      "reward_std": 0.009000815451145172,
+      "rewards//mean": 0.763427734375,
+      "rewards//std": 0.025083519518375397,
+      "step": 981
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1964,
+      "grad_norm": 1.3574923276901245,
+      "kl": 0.43050722032785416,
+      "learning_rate": 4.576136251515748e-06,
+      "loss": 0.0172,
+      "num_tokens": 8488528.0,
+      "reward": 0.745849609375,
+      "reward_std": 0.013025259599089622,
+      "rewards//mean": 0.745849609375,
+      "rewards//std": 0.030216630548238754,
+      "step": 982
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1966,
+      "grad_norm": 1.5071163177490234,
+      "kl": 0.39033103734254837,
+      "learning_rate": 4.575251924993926e-06,
+      "loss": 0.0156,
+      "num_tokens": 8497112.0,
+      "reward": 0.75579833984375,
+      "reward_std": 0.010598527267575264,
+      "rewards//mean": 0.75579833984375,
+      "rewards//std": 0.028314098715782166,
+      "step": 983
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1968,
+      "grad_norm": 1.5211485624313354,
+      "kl": 0.4381902143359184,
+      "learning_rate": 4.574366762560876e-06,
+      "loss": 0.0175,
+      "num_tokens": 8505752.0,
+      "reward": 0.76092529296875,
+      "reward_std": 0.008739031851291656,
+      "rewards//mean": 0.76092529296875,
+      "rewards//std": 0.028815951198339462,
+      "step": 984
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.197,
+      "grad_norm": 1.9539810419082642,
+      "kl": 0.4092373363673687,
+      "learning_rate": 4.573480764573143e-06,
+      "loss": 0.0164,
+      "num_tokens": 8514376.0,
+      "reward": 0.74774169921875,
+      "reward_std": 0.008132066577672958,
+      "rewards//mean": 0.74774169921875,
+      "rewards//std": 0.029592422768473625,
+      "step": 985
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1972,
+      "grad_norm": 1.4879189729690552,
+      "kl": 0.3980576805770397,
+      "learning_rate": 4.572593931387604e-06,
+      "loss": 0.0159,
+      "num_tokens": 8522960.0,
+      "reward": 0.724853515625,
+      "reward_std": 0.0099346823990345,
+      "rewards//mean": 0.724853515625,
+      "rewards//std": 0.03472910076379776,
+      "step": 986
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1974,
+      "grad_norm": 1.3883405923843384,
+      "kl": 0.4651208482682705,
+      "learning_rate": 4.571706263361479e-06,
+      "loss": 0.0186,
+      "num_tokens": 8531496.0,
+      "reward": 0.72979736328125,
+      "reward_std": 0.00940313097089529,
+      "rewards//mean": 0.72979736328125,
+      "rewards//std": 0.033729128539562225,
+      "step": 987
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1976,
+      "grad_norm": 1.4428088665008545,
+      "kl": 0.43703150749206543,
+      "learning_rate": 4.570817760852319e-06,
+      "loss": 0.0175,
+      "num_tokens": 8540232.0,
+      "reward": 0.7557373046875,
+      "reward_std": 0.006712545640766621,
+      "rewards//mean": 0.7557373046875,
+      "rewards//std": 0.029765138402581215,
+      "step": 988
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1978,
+      "grad_norm": 1.7286840677261353,
+      "kl": 0.4548807218670845,
+      "learning_rate": 4.569928424218012e-06,
+      "loss": 0.0182,
+      "num_tokens": 8548920.0,
+      "reward": 0.75628662109375,
+      "reward_std": 0.008855903521180153,
+      "rewards//mean": 0.75628662109375,
+      "rewards//std": 0.029729709029197693,
+      "step": 989
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.198,
+      "grad_norm": 1.4762978553771973,
+      "kl": 0.4822859615087509,
+      "learning_rate": 4.569038253816783e-06,
+      "loss": 0.0193,
+      "num_tokens": 8557688.0,
+      "reward": 0.784912109375,
+      "reward_std": 0.00827457383275032,
+      "rewards//mean": 0.784912109375,
+      "rewards//std": 0.02746196649968624,
+      "step": 990
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1982,
+      "grad_norm": 1.853870153427124,
+      "kl": 0.4191112294793129,
+      "learning_rate": 4.5681472500071935e-06,
+      "loss": 0.0168,
+      "num_tokens": 8566352.0,
+      "reward": 0.75384521484375,
+      "reward_std": 0.0067790113389492035,
+      "rewards//mean": 0.75384521484375,
+      "rewards//std": 0.024154985323548317,
+      "step": 991
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1984,
+      "grad_norm": 1.573944330215454,
+      "kl": 0.42034513130784035,
+      "learning_rate": 4.567255413148139e-06,
+      "loss": 0.0168,
+      "num_tokens": 8575064.0,
+      "reward": 0.7481689453125,
+      "reward_std": 0.009067408740520477,
+      "rewards//mean": 0.7481689453125,
+      "rewards//std": 0.02402908354997635,
+      "step": 992
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1986,
+      "grad_norm": 1.8001471757888794,
+      "kl": 0.4698791652917862,
+      "learning_rate": 4.566362743598851e-06,
+      "loss": 0.0188,
+      "num_tokens": 8583664.0,
+      "reward": 0.7425537109375,
+      "reward_std": 0.012988231144845486,
+      "rewards//mean": 0.7425537109375,
+      "rewards//std": 0.02789810486137867,
+      "step": 993
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1988,
+      "grad_norm": 1.660449743270874,
+      "kl": 0.46960631385445595,
+      "learning_rate": 4.565469241718896e-06,
+      "loss": 0.0188,
+      "num_tokens": 8592344.0,
+      "reward": 0.752685546875,
+      "reward_std": 0.005921890959143639,
+      "rewards//mean": 0.752685546875,
+      "rewards//std": 0.020586274564266205,
+      "step": 994
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.199,
+      "grad_norm": 1.4339450597763062,
+      "kl": 0.4214029908180237,
+      "learning_rate": 4.564574907868179e-06,
+      "loss": 0.0169,
+      "num_tokens": 8601024.0,
+      "reward": 0.7352294921875,
+      "reward_std": 0.007715367246419191,
+      "rewards//mean": 0.7352294921875,
+      "rewards//std": 0.03900335729122162,
+      "step": 995
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1992,
+      "grad_norm": 2.15255069732666,
+      "kl": 0.4501718766987324,
+      "learning_rate": 4.563679742406935e-06,
+      "loss": 0.018,
+      "num_tokens": 8609688.0,
+      "reward": 0.76776123046875,
+      "reward_std": 0.006294669583439827,
+      "rewards//mean": 0.76776123046875,
+      "rewards//std": 0.03227883577346802,
+      "step": 996
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1994,
+      "grad_norm": 1.7035322189331055,
+      "kl": 0.45111533999443054,
+      "learning_rate": 4.562783745695738e-06,
+      "loss": 0.018,
+      "num_tokens": 8618400.0,
+      "reward": 0.76800537109375,
+      "reward_std": 0.0073316022753715515,
+      "rewards//mean": 0.76800537109375,
+      "rewards//std": 0.028362175449728966,
+      "step": 997
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1996,
+      "grad_norm": 1.9999769926071167,
+      "kl": 0.4639809913933277,
+      "learning_rate": 4.561886918095495e-06,
+      "loss": 0.0186,
+      "num_tokens": 8627216.0,
+      "reward": 0.768798828125,
+      "reward_std": 0.011256873607635498,
+      "rewards//mean": 0.768798828125,
+      "rewards//std": 0.024358505383133888,
+      "step": 998
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1998,
+      "grad_norm": 1.600160837173462,
+      "kl": 0.44482381641864777,
+      "learning_rate": 4.560989259967447e-06,
+      "loss": 0.0178,
+      "num_tokens": 8635816.0,
+      "reward": 0.71075439453125,
+      "reward_std": 0.00779071357101202,
+      "rewards//mean": 0.71075439453125,
+      "rewards//std": 0.035272691398859024,
+      "step": 999
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2,
+      "grad_norm": 2.164851665496826,
+      "kl": 0.4851101525127888,
+      "learning_rate": 4.560090771673174e-06,
+      "loss": 0.0194,
+      "num_tokens": 8644496.0,
+      "reward": 0.7564697265625,
+      "reward_std": 0.014028170146048069,
+      "rewards//mean": 0.7564697265625,
+      "rewards//std": 0.03170565515756607,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 5000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}