diff --git "a/checkpoint-2000/trainer_state.json" "b/checkpoint-2000/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-2000/trainer_state.json"
@@ -0,0 +1,38034 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.4,
+  "eval_steps": 500,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0002,
+      "grad_norm": 3.150317668914795,
+      "kl": 0.00016507542022736743,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "num_tokens": 8600.0,
+      "reward": 0.6983642578125,
+      "reward_std": 0.012111281976103783,
+      "rewards//mean": 0.6983642578125,
+      "rewards//std": 0.04253233224153519,
+      "step": 1
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0004,
+      "grad_norm": 3.1340620517730713,
+      "kl": 9.68448439380154e-05,
+      "learning_rate": 2e-08,
+      "loss": 0.0,
+      "num_tokens": 17200.0,
+      "reward": 0.72998046875,
+      "reward_std": 0.016355562955141068,
+      "rewards//mean": 0.72998046875,
+      "rewards//std": 0.05969127640128136,
+      "step": 2
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0006,
+      "grad_norm": 3.628222703933716,
+      "kl": 0.0007720550274825655,
+      "learning_rate": 4e-08,
+      "loss": 0.0,
+      "num_tokens": 25872.0,
+      "reward": 0.737060546875,
+      "reward_std": 0.01231062039732933,
+      "rewards//mean": 0.737060546875,
+      "rewards//std": 0.04868406429886818,
+      "step": 3
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0008,
+      "grad_norm": 3.346497058868408,
+      "kl": 0.0007726833282504231,
+      "learning_rate": 6e-08,
+      "loss": 0.0,
+      "num_tokens": 34600.0,
+      "reward": 0.7144775390625,
+      "reward_std": 0.013249381445348263,
+      "rewards//mean": 0.7144775390625,
+      "rewards//std": 0.057925332337617874,
+      "step": 4
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.001,
+      "grad_norm": 3.1139121055603027,
+      "kl": 0.0006869801800348796,
+      "learning_rate": 8e-08,
+      "loss": 0.0,
+      "num_tokens": 43304.0,
+      "reward": 0.71942138671875,
+      "reward_std": 0.014566982164978981,
+      "rewards//mean": 0.71942138671875,
+      "rewards//std": 0.05822908505797386,
+      "step": 5
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0012,
+      "grad_norm": 3.0228867530822754,
+      "kl": 0.0007238477919599973,
+      "learning_rate": 1e-07,
+      "loss": 0.0,
+      "num_tokens": 51992.0,
+      "reward": 0.71722412109375,
+      "reward_std": 0.012525280006229877,
+      "rewards//mean": 0.71722412109375,
+      "rewards//std": 0.057698674499988556,
+      "step": 6
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0014,
+      "grad_norm": 3.3383941650390625,
+      "kl": 0.0007060512798489071,
+      "learning_rate": 1.2e-07,
+      "loss": 0.0,
+      "num_tokens": 60696.0,
+      "reward": 0.7244873046875,
+      "reward_std": 0.013975013047456741,
+      "rewards//mean": 0.7244873046875,
+      "rewards//std": 0.03846875578165054,
+      "step": 7
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0016,
+      "grad_norm": 3.2544026374816895,
+      "kl": 0.0007304435930564068,
+      "learning_rate": 1.4e-07,
+      "loss": 0.0,
+      "num_tokens": 69336.0,
+      "reward": 0.73199462890625,
+      "reward_std": 0.014917248860001564,
+      "rewards//mean": 0.73199462890625,
+      "rewards//std": 0.054437216371297836,
+      "step": 8
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0018,
+      "grad_norm": 3.2664318084716797,
+      "kl": 0.0006970661124796607,
+      "learning_rate": 1.6e-07,
+      "loss": 0.0,
+      "num_tokens": 78008.0,
+      "reward": 0.7119140625,
+      "reward_std": 0.016640130430459976,
+      "rewards//mean": 0.7119140625,
+      "rewards//std": 0.051102180033922195,
+      "step": 9
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.002,
+      "grad_norm": 3.30499529838562,
+      "kl": 0.0007583869155496359,
+      "learning_rate": 1.8e-07,
+      "loss": 0.0,
+      "num_tokens": 86648.0,
+      "reward": 0.67755126953125,
+      "reward_std": 0.015648527070879936,
+      "rewards//mean": 0.67755126953125,
+      "rewards//std": 0.04872909188270569,
+      "step": 10
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0022,
+      "grad_norm": 3.266948699951172,
+      "kl": 0.0007290949215530418,
+      "learning_rate": 2e-07,
+      "loss": 0.0,
+      "num_tokens": 95456.0,
+      "reward": 0.69329833984375,
+      "reward_std": 0.014868234284222126,
+      "rewards//mean": 0.69329833984375,
+      "rewards//std": 0.04043947160243988,
+      "step": 11
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0024,
+      "grad_norm": 3.2822325229644775,
+      "kl": 0.000768863974371925,
+      "learning_rate": 2.1999999999999998e-07,
+      "loss": 0.0,
+      "num_tokens": 104112.0,
+      "reward": 0.70721435546875,
+      "reward_std": 0.015602790750563145,
+      "rewards//mean": 0.70721435546875,
+      "rewards//std": 0.04803575947880745,
+      "step": 12
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0026,
+      "grad_norm": 3.1537024974823,
+      "kl": 0.000746092431654688,
+      "learning_rate": 2.4e-07,
+      "loss": 0.0,
+      "num_tokens": 112728.0,
+      "reward": 0.72723388671875,
+      "reward_std": 0.013450011610984802,
+      "rewards//mean": 0.72723388671875,
+      "rewards//std": 0.04304227605462074,
+      "step": 13
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0028,
+      "grad_norm": 3.1742215156555176,
+      "kl": 0.0006956102952244692,
+      "learning_rate": 2.6e-07,
+      "loss": 0.0,
+      "num_tokens": 121352.0,
+      "reward": 0.75360107421875,
+      "reward_std": 0.018189093098044395,
+      "rewards//mean": 0.75360107421875,
+      "rewards//std": 0.04565153270959854,
+      "step": 14
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.003,
+      "grad_norm": 3.1125144958496094,
+      "kl": 0.0006477628121501766,
+      "learning_rate": 2.8e-07,
+      "loss": 0.0,
+      "num_tokens": 130072.0,
+      "reward": 0.73187255859375,
+      "reward_std": 0.0172879658639431,
+      "rewards//mean": 0.73187255859375,
+      "rewards//std": 0.06254403293132782,
+      "step": 15
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0032,
+      "grad_norm": 3.3312599658966064,
+      "kl": 0.0007021335186436772,
+      "learning_rate": 3e-07,
+      "loss": 0.0,
+      "num_tokens": 138680.0,
+      "reward": 0.73480224609375,
+      "reward_std": 0.011995160952210426,
+      "rewards//mean": 0.73480224609375,
+      "rewards//std": 0.04787761718034744,
+      "step": 16
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0034,
+      "grad_norm": 3.2479970455169678,
+      "kl": 0.0006967349399928935,
+      "learning_rate": 3.2e-07,
+      "loss": 0.0,
+      "num_tokens": 147320.0,
+      "reward": 0.68505859375,
+      "reward_std": 0.013261607848107815,
+      "rewards//mean": 0.68505859375,
+      "rewards//std": 0.054848916828632355,
+      "step": 17
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0036,
+      "grad_norm": 3.3135740756988525,
+      "kl": 0.000760370421630796,
+      "learning_rate": 3.4000000000000003e-07,
+      "loss": 0.0,
+      "num_tokens": 155984.0,
+      "reward": 0.72137451171875,
+      "reward_std": 0.012660497799515724,
+      "rewards//mean": 0.72137451171875,
+      "rewards//std": 0.04701458662748337,
+      "step": 18
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0038,
+      "grad_norm": 3.2698936462402344,
+      "kl": 0.0007952585219754837,
+      "learning_rate": 3.6e-07,
+      "loss": 0.0,
+      "num_tokens": 164608.0,
+      "reward": 0.7022705078125,
+      "reward_std": 0.023350292816758156,
+      "rewards//mean": 0.7022705078125,
+      "rewards//std": 0.06180427595973015,
+      "step": 19
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.004,
+      "grad_norm": 3.1324961185455322,
+      "kl": 0.00072584531881148,
+      "learning_rate": 3.7999999999999996e-07,
+      "loss": 0.0,
+      "num_tokens": 173168.0,
+      "reward": 0.71453857421875,
+      "reward_std": 0.01809067279100418,
+      "rewards//mean": 0.71453857421875,
+      "rewards//std": 0.056603968143463135,
+      "step": 20
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0042,
+      "grad_norm": 3.1413869857788086,
+      "kl": 0.0007194258432718925,
+      "learning_rate": 4e-07,
+      "loss": 0.0,
+      "num_tokens": 181792.0,
+      "reward": 0.7171630859375,
+      "reward_std": 0.013818057253956795,
+      "rewards//mean": 0.7171630859375,
+      "rewards//std": 0.051577337086200714,
+      "step": 21
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0044,
+      "grad_norm": 3.1343071460723877,
+      "kl": 0.0007037941031740047,
+      "learning_rate": 4.1999999999999995e-07,
+      "loss": 0.0,
+      "num_tokens": 190392.0,
+      "reward": 0.731689453125,
+      "reward_std": 0.01459466852247715,
+      "rewards//mean": 0.731689453125,
+      "rewards//std": 0.03735656663775444,
+      "step": 22
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0046,
+      "grad_norm": 3.26190185546875,
+      "kl": 0.0007583637707284652,
+      "learning_rate": 4.3999999999999997e-07,
+      "loss": 0.0,
+      "num_tokens": 199080.0,
+      "reward": 0.698486328125,
+      "reward_std": 0.015130313113331795,
+      "rewards//mean": 0.698486328125,
+      "rewards//std": 0.04446122422814369,
+      "step": 23
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0048,
+      "grad_norm": 3.3528366088867188,
+      "kl": 0.0008030652170418762,
+      "learning_rate": 4.6e-07,
+      "loss": 0.0,
+      "num_tokens": 207720.0,
+      "reward": 0.72381591796875,
+      "reward_std": 0.015433166176080704,
+      "rewards//mean": 0.72381591796875,
+      "rewards//std": 0.053040314465761185,
+      "step": 24
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.005,
+      "grad_norm": 3.0460476875305176,
+      "kl": 0.0007812085095793009,
+      "learning_rate": 4.8e-07,
+      "loss": 0.0,
+      "num_tokens": 216296.0,
+      "reward": 0.70770263671875,
+      "reward_std": 0.014261599630117416,
+      "rewards//mean": 0.70770263671875,
+      "rewards//std": 0.05814167112112045,
+      "step": 25
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0052,
+      "grad_norm": 3.184849500656128,
+      "kl": 0.0007379321614280343,
+      "learning_rate": 5e-07,
+      "loss": 0.0,
+      "num_tokens": 224968.0,
+      "reward": 0.74749755859375,
+      "reward_std": 0.01661759801208973,
+      "rewards//mean": 0.74749755859375,
+      "rewards//std": 0.05052240192890167,
+      "step": 26
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0054,
+      "grad_norm": 3.071376085281372,
+      "kl": 0.0007392473853542469,
+      "learning_rate": 5.2e-07,
+      "loss": 0.0,
+      "num_tokens": 233528.0,
+      "reward": 0.67926025390625,
+      "reward_std": 0.015911182388663292,
+      "rewards//mean": 0.67926025390625,
+      "rewards//std": 0.07468271255493164,
+      "step": 27
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0056,
+      "grad_norm": 3.1181528568267822,
+      "kl": 0.0007620643737027422,
+      "learning_rate": 5.4e-07,
+      "loss": 0.0,
+      "num_tokens": 242280.0,
+      "reward": 0.68609619140625,
+      "reward_std": 0.015002391301095486,
+      "rewards//mean": 0.68609619140625,
+      "rewards//std": 0.05366988107562065,
+      "step": 28
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0058,
+      "grad_norm": 3.293017864227295,
+      "kl": 0.0007734682003501803,
+      "learning_rate": 5.6e-07,
+      "loss": 0.0,
+      "num_tokens": 250976.0,
+      "reward": 0.7156982421875,
+      "reward_std": 0.011221669614315033,
+      "rewards//mean": 0.7156982421875,
+      "rewards//std": 0.050113748759031296,
+      "step": 29
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.006,
+      "grad_norm": 3.1560003757476807,
+      "kl": 0.0007398290908895433,
+      "learning_rate": 5.8e-07,
+      "loss": 0.0,
+      "num_tokens": 259632.0,
+      "reward": 0.703857421875,
+      "reward_std": 0.015436184592545033,
+      "rewards//mean": 0.703857421875,
+      "rewards//std": 0.04807832092046738,
+      "step": 30
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0062,
+      "grad_norm": 2.963697671890259,
+      "kl": 0.0007791937241563573,
+      "learning_rate": 6e-07,
+      "loss": 0.0,
+      "num_tokens": 268208.0,
+      "reward": 0.74041748046875,
+      "reward_std": 0.018390730023384094,
+      "rewards//mean": 0.74041748046875,
+      "rewards//std": 0.045399487018585205,
+      "step": 31
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0064,
+      "grad_norm": 3.0100553035736084,
+      "kl": 0.0008315800077980384,
+      "learning_rate": 6.2e-07,
+      "loss": 0.0,
+      "num_tokens": 276824.0,
+      "reward": 0.71026611328125,
+      "reward_std": 0.015266066417098045,
+      "rewards//mean": 0.71026611328125,
+      "rewards//std": 0.05298949033021927,
+      "step": 32
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0066,
+      "grad_norm": 3.014559268951416,
+      "kl": 0.0007704183590249158,
+      "learning_rate": 6.4e-07,
+      "loss": 0.0,
+      "num_tokens": 285520.0,
+      "reward": 0.73516845703125,
+      "reward_std": 0.01908334344625473,
+      "rewards//mean": 0.73516845703125,
+      "rewards//std": 0.050059158354997635,
+      "step": 33
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0068,
+      "grad_norm": 3.116197109222412,
+      "kl": 0.00073820207035169,
+      "learning_rate": 6.6e-07,
+      "loss": 0.0,
+      "num_tokens": 294144.0,
+      "reward": 0.70306396484375,
+      "reward_std": 0.015610162168741226,
+      "rewards//mean": 0.70306396484375,
+      "rewards//std": 0.050647489726543427,
+      "step": 34
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.007,
+      "grad_norm": 3.0357425212860107,
+      "kl": 0.000833665588288568,
+      "learning_rate": 6.800000000000001e-07,
+      "loss": 0.0,
+      "num_tokens": 302888.0,
+      "reward": 0.73577880859375,
+      "reward_std": 0.012224329635500908,
+      "rewards//mean": 0.73577880859375,
+      "rewards//std": 0.04008077085018158,
+      "step": 35
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0072,
+      "grad_norm": 3.2299728393554688,
+      "kl": 0.0008243657503044233,
+      "learning_rate": 7e-07,
+      "loss": 0.0,
+      "num_tokens": 311504.0,
+      "reward": 0.711181640625,
+      "reward_std": 0.014584844931960106,
+      "rewards//mean": 0.711181640625,
+      "rewards//std": 0.0634838342666626,
+      "step": 36
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0074,
+      "grad_norm": 3.1693902015686035,
+      "kl": 0.0008555862732464448,
+      "learning_rate": 7.2e-07,
+      "loss": 0.0,
+      "num_tokens": 320208.0,
+      "reward": 0.7293701171875,
+      "reward_std": 0.016974415630102158,
+      "rewards//mean": 0.7293701171875,
+      "rewards//std": 0.045011255890131,
+      "step": 37
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0076,
+      "grad_norm": 3.0704615116119385,
+      "kl": 0.0007757811254123226,
+      "learning_rate": 7.4e-07,
+      "loss": 0.0,
+      "num_tokens": 328904.0,
+      "reward": 0.707763671875,
+      "reward_std": 0.011619615368545055,
+      "rewards//mean": 0.707763671875,
+      "rewards//std": 0.0381837859749794,
+      "step": 38
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0078,
+      "grad_norm": 3.3403103351593018,
+      "kl": 0.0008441023965133354,
+      "learning_rate": 7.599999999999999e-07,
+      "loss": 0.0,
+      "num_tokens": 337632.0,
+      "reward": 0.7027587890625,
+      "reward_std": 0.014964539557695389,
+      "rewards//mean": 0.7027587890625,
+      "rewards//std": 0.06321561336517334,
+      "step": 39
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.008,
+      "grad_norm": 3.3247711658477783,
+      "kl": 0.0009583122737240046,
+      "learning_rate": 7.799999999999999e-07,
+      "loss": 0.0,
+      "num_tokens": 346264.0,
+      "reward": 0.6885986328125,
+      "reward_std": 0.017985574901103973,
+      "rewards//mean": 0.6885986328125,
+      "rewards//std": 0.048691995441913605,
+      "step": 40
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0082,
+      "grad_norm": 3.1681082248687744,
+      "kl": 0.000914801872568205,
+      "learning_rate": 8e-07,
+      "loss": 0.0,
+      "num_tokens": 354912.0,
+      "reward": 0.72589111328125,
+      "reward_std": 0.014821438118815422,
+      "rewards//mean": 0.72589111328125,
+      "rewards//std": 0.06690064072608948,
+      "step": 41
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0084,
+      "grad_norm": 3.135946750640869,
+      "kl": 0.0009579255347489379,
+      "learning_rate": 8.199999999999999e-07,
+      "loss": 0.0,
+      "num_tokens": 363544.0,
+      "reward": 0.74017333984375,
+      "reward_std": 0.013390760868787766,
+      "rewards//mean": 0.74017333984375,
+      "rewards//std": 0.04873934015631676,
+      "step": 42
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0086,
+      "grad_norm": 3.5682313442230225,
+      "kl": 0.0010489100313861854,
+      "learning_rate": 8.399999999999999e-07,
+      "loss": 0.0,
+      "num_tokens": 372144.0,
+      "reward": 0.73565673828125,
+      "reward_std": 0.020230459049344063,
+      "rewards//mean": 0.73565673828125,
+      "rewards//std": 0.05228316783905029,
+      "step": 43
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0088,
+      "grad_norm": 3.349270820617676,
+      "kl": 0.0009648135091993026,
+      "learning_rate": 8.599999999999999e-07,
+      "loss": 0.0,
+      "num_tokens": 380848.0,
+      "reward": 0.73638916015625,
+      "reward_std": 0.013494866900146008,
+      "rewards//mean": 0.73638916015625,
+      "rewards//std": 0.049878910183906555,
+      "step": 44
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.009,
+      "grad_norm": 3.222097396850586,
+      "kl": 0.0010578580695437267,
+      "learning_rate": 8.799999999999999e-07,
+      "loss": 0.0,
+      "num_tokens": 389504.0,
+      "reward": 0.67010498046875,
+      "reward_std": 0.013411211781203747,
+      "rewards//mean": 0.67010498046875,
+      "rewards//std": 0.052416760474443436,
+      "step": 45
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0092,
+      "grad_norm": 3.1397805213928223,
+      "kl": 0.0009474716134718619,
+      "learning_rate": 9e-07,
+      "loss": 0.0,
+      "num_tokens": 398200.0,
+      "reward": 0.72802734375,
+      "reward_std": 0.011726010590791702,
+      "rewards//mean": 0.72802734375,
+      "rewards//std": 0.06396511197090149,
+      "step": 46
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0094,
+      "grad_norm": 3.248054027557373,
+      "kl": 0.0010289600759278983,
+      "learning_rate": 9.2e-07,
+      "loss": 0.0,
+      "num_tokens": 406848.0,
+      "reward": 0.69805908203125,
+      "reward_std": 0.018465079367160797,
+      "rewards//mean": 0.69805908203125,
+      "rewards//std": 0.05887490510940552,
+      "step": 47
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0096,
+      "grad_norm": 3.210569143295288,
+      "kl": 0.0010431300543132238,
+      "learning_rate": 9.399999999999999e-07,
+      "loss": 0.0,
+      "num_tokens": 415520.0,
+      "reward": 0.7083740234375,
+      "reward_std": 0.014447823166847229,
+      "rewards//mean": 0.7083740234375,
+      "rewards//std": 0.06324434280395508,
+      "step": 48
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0098,
+      "grad_norm": 3.0604236125946045,
+      "kl": 0.0009561540064169094,
+      "learning_rate": 9.6e-07,
+      "loss": 0.0,
+      "num_tokens": 424088.0,
+      "reward": 0.7181396484375,
+      "reward_std": 0.016763266175985336,
+      "rewards//mean": 0.7181396484375,
+      "rewards//std": 0.06502308696508408,
+      "step": 49
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.01,
+      "grad_norm": 3.190124988555908,
+      "kl": 0.0010131369563168846,
+      "learning_rate": 9.8e-07,
+      "loss": 0.0,
+      "num_tokens": 432696.0,
+      "reward": 0.703125,
+      "reward_std": 0.014796003699302673,
+      "rewards//mean": 0.703125,
+      "rewards//std": 0.05059729143977165,
+      "step": 50
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0102,
+      "grad_norm": 2.9405922889709473,
+      "kl": 0.0010682813081075437,
+      "learning_rate": 1e-06,
+      "loss": 0.0,
+      "num_tokens": 441352.0,
+      "reward": 0.7259521484375,
+      "reward_std": 0.01541807409375906,
+      "rewards//mean": 0.7259521484375,
+      "rewards//std": 0.03761079162359238,
+      "step": 51
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0104,
+      "grad_norm": 3.0414485931396484,
+      "kl": 0.0011730788828572258,
+      "learning_rate": 9.999998993000298e-07,
+      "loss": 0.0,
+      "num_tokens": 449960.0,
+      "reward": 0.736572265625,
+      "reward_std": 0.01379593275487423,
+      "rewards//mean": 0.736572265625,
+      "rewards//std": 0.05054400861263275,
+      "step": 52
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0106,
+      "grad_norm": 3.2517385482788086,
+      "kl": 0.001291139968088828,
+      "learning_rate": 9.999995972001601e-07,
+      "loss": 0.0001,
+      "num_tokens": 458512.0,
+      "reward": 0.720703125,
+      "reward_std": 0.017797932028770447,
+      "rewards//mean": 0.720703125,
+      "rewards//std": 0.04921412467956543,
+      "step": 53
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0108,
+      "grad_norm": 3.3084728717803955,
+      "kl": 0.0012594063809956424,
+      "learning_rate": 9.999990937005123e-07,
+      "loss": 0.0001,
+      "num_tokens": 467048.0,
+      "reward": 0.7257080078125,
+      "reward_std": 0.013701392337679863,
+      "rewards//mean": 0.7257080078125,
+      "rewards//std": 0.04821210727095604,
+      "step": 54
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.011,
+      "grad_norm": 3.0424487590789795,
+      "kl": 0.00110630490235053,
+      "learning_rate": 9.999983888012896e-07,
+      "loss": 0.0,
+      "num_tokens": 475728.0,
+      "reward": 0.7081298828125,
+      "reward_std": 0.017631450667977333,
+      "rewards//mean": 0.7081298828125,
+      "rewards//std": 0.07252521812915802,
+      "step": 55
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0112,
+      "grad_norm": 3.2253611087799072,
+      "kl": 0.001307873819314409,
+      "learning_rate": 9.999974825027754e-07,
+      "loss": 0.0001,
+      "num_tokens": 484360.0,
+      "reward": 0.72161865234375,
+      "reward_std": 0.012384735979139805,
+      "rewards//mean": 0.72161865234375,
+      "rewards//std": 0.054741695523262024,
+      "step": 56
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0114,
+      "grad_norm": 3.382544994354248,
+      "kl": 0.0015232005753205158,
+      "learning_rate": 9.999963748053354e-07,
+      "loss": 0.0001,
+      "num_tokens": 493000.0,
+      "reward": 0.73602294921875,
+      "reward_std": 0.016367167234420776,
+      "rewards//mean": 0.73602294921875,
+      "rewards//std": 0.05018298700451851,
+      "step": 57
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0116,
+      "grad_norm": 3.4117281436920166,
+      "kl": 0.0017642830134718679,
+      "learning_rate": 9.99995065709415e-07,
+      "loss": 0.0001,
+      "num_tokens": 501632.0,
+      "reward": 0.72265625,
+      "reward_std": 0.014317058026790619,
+      "rewards//mean": 0.72265625,
+      "rewards//std": 0.04878905043005943,
+      "step": 58
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0118,
+      "grad_norm": 3.2561867237091064,
+      "kl": 0.0014012602186994627,
+      "learning_rate": 9.999935552155421e-07,
+      "loss": 0.0001,
+      "num_tokens": 510288.0,
+      "reward": 0.726318359375,
+      "reward_std": 0.014060637913644314,
+      "rewards//mean": 0.726318359375,
+      "rewards//std": 0.04617677628993988,
+      "step": 59
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.012,
+      "grad_norm": 3.177072286605835,
+      "kl": 0.0016652587437420152,
+      "learning_rate": 9.99991843324325e-07,
+      "loss": 0.0001,
+      "num_tokens": 518952.0,
+      "reward": 0.71875,
+      "reward_std": 0.018966306000947952,
+      "rewards//mean": 0.71875,
+      "rewards//std": 0.05035256966948509,
+      "step": 60
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0122,
+      "grad_norm": 3.005466938018799,
+      "kl": 0.0016885494842426851,
+      "learning_rate": 9.999899300364532e-07,
+      "loss": 0.0001,
+      "num_tokens": 527520.0,
+      "reward": 0.71490478515625,
+      "reward_std": 0.01147861871868372,
+      "rewards//mean": 0.71490478515625,
+      "rewards//std": 0.04259505867958069,
+      "step": 61
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0124,
+      "grad_norm": 3.3473289012908936,
+      "kl": 0.0016567637867410667,
+      "learning_rate": 9.999878153526972e-07,
+      "loss": 0.0001,
+      "num_tokens": 536112.0,
+      "reward": 0.70928955078125,
+      "reward_std": 0.018018949776887894,
+      "rewards//mean": 0.70928955078125,
+      "rewards//std": 0.05387454107403755,
+      "step": 62
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0126,
+      "grad_norm": 3.2961602210998535,
+      "kl": 0.002063060695945751,
+      "learning_rate": 9.999854992739093e-07,
+      "loss": 0.0001,
+      "num_tokens": 544736.0,
+      "reward": 0.6990966796875,
+      "reward_std": 0.012440960854291916,
+      "rewards//mean": 0.6990966796875,
+      "rewards//std": 0.04440177232027054,
+      "step": 63
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0128,
+      "grad_norm": 3.2076351642608643,
+      "kl": 0.0022169264702824876,
+      "learning_rate": 9.999829818010219e-07,
+      "loss": 0.0001,
+      "num_tokens": 553408.0,
+      "reward": 0.71697998046875,
+      "reward_std": 0.015242921188473701,
+      "rewards//mean": 0.71697998046875,
+      "rewards//std": 0.04653756693005562,
+      "step": 64
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.013,
+      "grad_norm": 3.063619375228882,
+      "kl": 0.00204440436209552,
+      "learning_rate": 9.999802629350491e-07,
+      "loss": 0.0001,
+      "num_tokens": 562064.0,
+      "reward": 0.72161865234375,
+      "reward_std": 0.0141282444819808,
+      "rewards//mean": 0.72161865234375,
+      "rewards//std": 0.046424560248851776,
+      "step": 65
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0132,
+      "grad_norm": 2.984975576400757,
+      "kl": 0.002333598895347677,
+      "learning_rate": 9.999773426770863e-07,
+      "loss": 0.0001,
+      "num_tokens": 570664.0,
+      "reward": 0.71575927734375,
+      "reward_std": 0.011713966727256775,
+      "rewards//mean": 0.71575927734375,
+      "rewards//std": 0.04769832640886307,
+      "step": 66
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0134,
+      "grad_norm": 3.0676863193511963,
+      "kl": 0.002809927915222943,
+      "learning_rate": 9.999742210283097e-07,
+      "loss": 0.0001,
+      "num_tokens": 579360.0,
+      "reward": 0.71435546875,
+      "reward_std": 0.011259706690907478,
+      "rewards//mean": 0.71435546875,
+      "rewards//std": 0.048482783138751984,
+      "step": 67
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0136,
+      "grad_norm": 2.854644298553467,
+      "kl": 0.0022947356701479293,
+      "learning_rate": 9.999708979899767e-07,
+      "loss": 0.0001,
+      "num_tokens": 587992.0,
+      "reward": 0.7469482421875,
+      "reward_std": 0.015341498889029026,
+      "rewards//mean": 0.7469482421875,
+      "rewards//std": 0.039588894695043564,
+      "step": 68
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0138,
+      "grad_norm": 2.998915195465088,
+      "kl": 0.0029584673466160893,
+      "learning_rate": 9.999673735634259e-07,
+      "loss": 0.0001,
+      "num_tokens": 596608.0,
+      "reward": 0.69549560546875,
+      "reward_std": 0.01312573067843914,
+      "rewards//mean": 0.69549560546875,
+      "rewards//std": 0.036401353776454926,
+      "step": 69
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.014,
+      "grad_norm": 3.1461427211761475,
+      "kl": 0.003096007196290884,
+      "learning_rate": 9.999636477500764e-07,
+      "loss": 0.0001,
+      "num_tokens": 605248.0,
+      "reward": 0.69873046875,
+      "reward_std": 0.02277829684317112,
+      "rewards//mean": 0.69873046875,
+      "rewards//std": 0.044763218611478806,
+      "step": 70
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0142,
+      "grad_norm": 3.069575071334839,
+      "kl": 0.003110720237600617,
+      "learning_rate": 9.999597205514296e-07,
+      "loss": 0.0001,
+      "num_tokens": 613824.0,
+      "reward": 0.69732666015625,
+      "reward_std": 0.018011249601840973,
+      "rewards//mean": 0.69732666015625,
+      "rewards//std": 0.04875921458005905,
+      "step": 71
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0144,
+      "grad_norm": 3.1159253120422363,
+      "kl": 0.00321734477620339,
+      "learning_rate": 9.999555919690672e-07,
+      "loss": 0.0001,
+      "num_tokens": 622328.0,
+      "reward": 0.74285888671875,
+      "reward_std": 0.016840215772390366,
+      "rewards//mean": 0.74285888671875,
+      "rewards//std": 0.06359697133302689,
+      "step": 72
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0146,
+      "grad_norm": 3.0868067741394043,
+      "kl": 0.003257128431869205,
+      "learning_rate": 9.99951262004652e-07,
+      "loss": 0.0001,
+      "num_tokens": 630976.0,
+      "reward": 0.6861572265625,
+      "reward_std": 0.017197417095303535,
+      "rewards//mean": 0.6861572265625,
+      "rewards//std": 0.04307975620031357,
+      "step": 73
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0148,
+      "grad_norm": 2.944159984588623,
+      "kl": 0.0035032854502787814,
+      "learning_rate": 9.999467306599285e-07,
+      "loss": 0.0001,
+      "num_tokens": 639624.0,
+      "reward": 0.73211669921875,
+      "reward_std": 0.012926152907311916,
+      "rewards//mean": 0.73211669921875,
+      "rewards//std": 0.03628389537334442,
+      "step": 74
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.015,
+      "grad_norm": 3.0424153804779053,
+      "kl": 0.004879061205429025,
+      "learning_rate": 9.999419979367214e-07,
+      "loss": 0.0002,
+      "num_tokens": 648320.0,
+      "reward": 0.704345703125,
+      "reward_std": 0.014891589060425758,
+      "rewards//mean": 0.704345703125,
+      "rewards//std": 0.057871606200933456,
+      "step": 75
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0152,
+      "grad_norm": 3.2213408946990967,
+      "kl": 0.004950453294441104,
+      "learning_rate": 9.999370638369376e-07,
+      "loss": 0.0002,
+      "num_tokens": 657080.0,
+      "reward": 0.7288818359375,
+      "reward_std": 0.014744549989700317,
+      "rewards//mean": 0.7288818359375,
+      "rewards//std": 0.04380178079009056,
+      "step": 76
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0154,
+      "grad_norm": 3.053184747695923,
+      "kl": 0.0044919641222804785,
+      "learning_rate": 9.99931928362564e-07,
+      "loss": 0.0002,
+      "num_tokens": 665720.0,
+      "reward": 0.6983642578125,
+      "reward_std": 0.0167398564517498,
+      "rewards//mean": 0.6983642578125,
+      "rewards//std": 0.06560251861810684,
+      "step": 77
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0156,
+      "grad_norm": 3.2201783657073975,
+      "kl": 0.00456669855338987,
+      "learning_rate": 9.999265915156696e-07,
+      "loss": 0.0002,
+      "num_tokens": 674336.0,
+      "reward": 0.7213134765625,
+      "reward_std": 0.014512969180941582,
+      "rewards//mean": 0.7213134765625,
+      "rewards//std": 0.05112217366695404,
+      "step": 78
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0158,
+      "grad_norm": 3.2414867877960205,
+      "kl": 0.004756899201311171,
+      "learning_rate": 9.999210532984038e-07,
+      "loss": 0.0002,
+      "num_tokens": 682968.0,
+      "reward": 0.68731689453125,
+      "reward_std": 0.01768694445490837,
+      "rewards//mean": 0.68731689453125,
+      "rewards//std": 0.06721082329750061,
+      "step": 79
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.016,
+      "grad_norm": 2.93839430809021,
+      "kl": 0.005078462738310918,
+      "learning_rate": 9.999153137129977e-07,
+      "loss": 0.0002,
+      "num_tokens": 691640.0,
+      "reward": 0.740478515625,
+      "reward_std": 0.011827691458165646,
+      "rewards//mean": 0.740478515625,
+      "rewards//std": 0.06183329224586487,
+      "step": 80
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0162,
+      "grad_norm": 3.0919079780578613,
+      "kl": 0.00472830337821506,
+      "learning_rate": 9.999093727617628e-07,
+      "loss": 0.0002,
+      "num_tokens": 700264.0,
+      "reward": 0.7027587890625,
+      "reward_std": 0.015644310042262077,
+      "rewards//mean": 0.7027587890625,
+      "rewards//std": 0.039406463503837585,
+      "step": 81
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0164,
+      "grad_norm": 3.026963710784912,
+      "kl": 0.005380780916311778,
+      "learning_rate": 9.999032304470924e-07,
+      "loss": 0.0002,
+      "num_tokens": 708984.0,
+      "reward": 0.73486328125,
+      "reward_std": 0.014729069545865059,
+      "rewards//mean": 0.73486328125,
+      "rewards//std": 0.043728627264499664,
+      "step": 82
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0166,
+      "grad_norm": 3.0130741596221924,
+      "kl": 0.006531667575472966,
+      "learning_rate": 9.998968867714608e-07,
+      "loss": 0.0003,
+      "num_tokens": 717568.0,
+      "reward": 0.72998046875,
+      "reward_std": 0.012801310047507286,
+      "rewards//mean": 0.72998046875,
+      "rewards//std": 0.04702718183398247,
+      "step": 83
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0168,
+      "grad_norm": 3.0457839965820312,
+      "kl": 0.0055532880942337215,
+      "learning_rate": 9.998903417374226e-07,
+      "loss": 0.0002,
+      "num_tokens": 726304.0,
+      "reward": 0.710693359375,
+      "reward_std": 0.01631096936762333,
+      "rewards//mean": 0.710693359375,
+      "rewards//std": 0.04329654574394226,
+      "step": 84
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.017,
+      "grad_norm": 3.1099348068237305,
+      "kl": 0.005788850947283208,
+      "learning_rate": 9.998835953476147e-07,
+      "loss": 0.0002,
+      "num_tokens": 735000.0,
+      "reward": 0.7353515625,
+      "reward_std": 0.016055503860116005,
+      "rewards//mean": 0.7353515625,
+      "rewards//std": 0.05147054046392441,
+      "step": 85
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0172,
+      "grad_norm": 3.253512144088745,
+      "kl": 0.007270405476447195,
+      "learning_rate": 9.998766476047545e-07,
+      "loss": 0.0003,
+      "num_tokens": 743648.0,
+      "reward": 0.7015380859375,
+      "reward_std": 0.01626693084836006,
+      "rewards//mean": 0.7015380859375,
+      "rewards//std": 0.06402271240949631,
+      "step": 86
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0174,
+      "grad_norm": 3.139979600906372,
+      "kl": 0.008355276426300406,
+      "learning_rate": 9.998694985116404e-07,
+      "loss": 0.0003,
+      "num_tokens": 752416.0,
+      "reward": 0.7178955078125,
+      "reward_std": 0.013003869913518429,
+      "rewards//mean": 0.7178955078125,
+      "rewards//std": 0.06749863922595978,
+      "step": 87
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0176,
+      "grad_norm": 3.1358847618103027,
+      "kl": 0.007668178965104744,
+      "learning_rate": 9.99862148071152e-07,
+      "loss": 0.0003,
+      "num_tokens": 761040.0,
+      "reward": 0.73675537109375,
+      "reward_std": 0.018933400511741638,
+      "rewards//mean": 0.73675537109375,
+      "rewards//std": 0.06281549483537674,
+      "step": 88
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0178,
+      "grad_norm": 3.1049060821533203,
+      "kl": 0.007956226007081568,
+      "learning_rate": 9.998545962862501e-07,
+      "loss": 0.0003,
+      "num_tokens": 769656.0,
+      "reward": 0.743408203125,
+      "reward_std": 0.016416165977716446,
+      "rewards//mean": 0.743408203125,
+      "rewards//std": 0.05655932053923607,
+      "step": 89
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.018,
+      "grad_norm": 2.9268035888671875,
+      "kl": 0.008059439191129059,
+      "learning_rate": 9.998468431599767e-07,
+      "loss": 0.0003,
+      "num_tokens": 778248.0,
+      "reward": 0.7052001953125,
+      "reward_std": 0.016768010333180428,
+      "rewards//mean": 0.7052001953125,
+      "rewards//std": 0.022466329857707024,
+      "step": 90
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0182,
+      "grad_norm": 3.3397057056427,
+      "kl": 0.010496469039935619,
+      "learning_rate": 9.998388886954545e-07,
+      "loss": 0.0004,
+      "num_tokens": 786856.0,
+      "reward": 0.69293212890625,
+      "reward_std": 0.014789804816246033,
+      "rewards//mean": 0.69293212890625,
+      "rewards//std": 0.069489985704422,
+      "step": 91
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0184,
+      "grad_norm": 3.0880322456359863,
+      "kl": 0.009906304272590205,
+      "learning_rate": 9.998307328958877e-07,
+      "loss": 0.0004,
+      "num_tokens": 795544.0,
+      "reward": 0.6934814453125,
+      "reward_std": 0.015139150433242321,
+      "rewards//mean": 0.6934814453125,
+      "rewards//std": 0.06462985277175903,
+      "step": 92
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0186,
+      "grad_norm": 3.5390796661376953,
+      "kl": 0.010611087462166324,
+      "learning_rate": 9.998223757645617e-07,
+      "loss": 0.0004,
+      "num_tokens": 804104.0,
+      "reward": 0.7440185546875,
+      "reward_std": 0.013973723165690899,
+      "rewards//mean": 0.7440185546875,
+      "rewards//std": 0.04960612207651138,
+      "step": 93
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0188,
+      "grad_norm": 3.8798980712890625,
+      "kl": 0.011519388906890526,
+      "learning_rate": 9.998138173048423e-07,
+      "loss": 0.0005,
+      "num_tokens": 812768.0,
+      "reward": 0.7451171875,
+      "reward_std": 0.015056892298161983,
+      "rewards//mean": 0.7451171875,
+      "rewards//std": 0.0555574931204319,
+      "step": 94
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.019,
+      "grad_norm": 3.1219942569732666,
+      "kl": 0.009766744391527027,
+      "learning_rate": 9.99805057520177e-07,
+      "loss": 0.0004,
+      "num_tokens": 821400.0,
+      "reward": 0.74365234375,
+      "reward_std": 0.013678168877959251,
+      "rewards//mean": 0.74365234375,
+      "rewards//std": 0.046017181128263474,
+      "step": 95
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0192,
+      "grad_norm": 2.997413396835327,
+      "kl": 0.00959825367317535,
+      "learning_rate": 9.997960964140945e-07,
+      "loss": 0.0004,
+      "num_tokens": 829952.0,
+      "reward": 0.7000732421875,
+      "reward_std": 0.016240671277046204,
+      "rewards//mean": 0.7000732421875,
+      "rewards//std": 0.03586537390947342,
+      "step": 96
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0194,
+      "grad_norm": 3.278810501098633,
+      "kl": 0.011237991857342422,
+      "learning_rate": 9.99786933990204e-07,
+      "loss": 0.0004,
+      "num_tokens": 838520.0,
+      "reward": 0.71533203125,
+      "reward_std": 0.011809727177023888,
+      "rewards//mean": 0.71533203125,
+      "rewards//std": 0.04853271320462227,
+      "step": 97
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0196,
+      "grad_norm": 2.8756959438323975,
+      "kl": 0.010547184152528644,
+      "learning_rate": 9.997775702521965e-07,
+      "loss": 0.0004,
+      "num_tokens": 847128.0,
+      "reward": 0.707275390625,
+      "reward_std": 0.017715217545628548,
+      "rewards//mean": 0.707275390625,
+      "rewards//std": 0.07538240402936935,
+      "step": 98
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0198,
+      "grad_norm": 3.121904134750366,
+      "kl": 0.012521670694695786,
+      "learning_rate": 9.997680052038434e-07,
+      "loss": 0.0005,
+      "num_tokens": 855824.0,
+      "reward": 0.69482421875,
+      "reward_std": 0.01390664093196392,
+      "rewards//mean": 0.69482421875,
+      "rewards//std": 0.05070488154888153,
+      "step": 99
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.02,
+      "grad_norm": 3.230713129043579,
+      "kl": 0.012724154919851571,
+      "learning_rate": 9.997582388489973e-07,
+      "loss": 0.0005,
+      "num_tokens": 864520.0,
+      "reward": 0.71173095703125,
+      "reward_std": 0.015987861901521683,
+      "rewards//mean": 0.71173095703125,
+      "rewards//std": 0.08442246913909912,
+      "step": 100
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0202,
+      "grad_norm": 3.1842575073242188,
+      "kl": 0.014311595150502399,
+      "learning_rate": 9.997482711915925e-07,
+      "loss": 0.0006,
+      "num_tokens": 873152.0,
+      "reward": 0.70654296875,
+      "reward_std": 0.017925892025232315,
+      "rewards//mean": 0.70654296875,
+      "rewards//std": 0.0551220178604126,
+      "step": 101
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0204,
+      "grad_norm": 2.9082493782043457,
+      "kl": 0.010199115553405136,
+      "learning_rate": 9.99738102235644e-07,
+      "loss": 0.0004,
+      "num_tokens": 881824.0,
+      "reward": 0.7142333984375,
+      "reward_std": 0.014553748071193695,
+      "rewards//mean": 0.7142333984375,
+      "rewards//std": 0.04478874057531357,
+      "step": 102
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0206,
+      "grad_norm": 3.063404083251953,
+      "kl": 0.012870212813140824,
+      "learning_rate": 9.997277319852474e-07,
+      "loss": 0.0005,
+      "num_tokens": 890368.0,
+      "reward": 0.748779296875,
+      "reward_std": 0.011821390129625797,
+      "rewards//mean": 0.748779296875,
+      "rewards//std": 0.04491109028458595,
+      "step": 103
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0208,
+      "grad_norm": 3.2403464317321777,
+      "kl": 0.015398608316900209,
+      "learning_rate": 9.997171604445802e-07,
+      "loss": 0.0006,
+      "num_tokens": 899128.0,
+      "reward": 0.762451171875,
+      "reward_std": 0.019849859178066254,
+      "rewards//mean": 0.762451171875,
+      "rewards//std": 0.04154951870441437,
+      "step": 104
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.021,
+      "grad_norm": 3.1215336322784424,
+      "kl": 0.012001491704722866,
+      "learning_rate": 9.997063876179007e-07,
+      "loss": 0.0005,
+      "num_tokens": 907808.0,
+      "reward": 0.71649169921875,
+      "reward_std": 0.015484604984521866,
+      "rewards//mean": 0.71649169921875,
+      "rewards//std": 0.05297234654426575,
+      "step": 105
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0212,
+      "grad_norm": 2.9595353603363037,
+      "kl": 0.015639638353604823,
+      "learning_rate": 9.996954135095478e-07,
+      "loss": 0.0006,
+      "num_tokens": 916384.0,
+      "reward": 0.74432373046875,
+      "reward_std": 0.016129087656736374,
+      "rewards//mean": 0.74432373046875,
+      "rewards//std": 0.047977738082408905,
+      "step": 106
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0214,
+      "grad_norm": 2.8981754779815674,
+      "kl": 0.016912667255382985,
+      "learning_rate": 9.996842381239422e-07,
+      "loss": 0.0007,
+      "num_tokens": 925000.0,
+      "reward": 0.7364501953125,
+      "reward_std": 0.013939950615167618,
+      "rewards//mean": 0.7364501953125,
+      "rewards//std": 0.031581275165081024,
+      "step": 107
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0216,
+      "grad_norm": 2.97182559967041,
+      "kl": 0.01626437672530301,
+      "learning_rate": 9.996728614655853e-07,
+      "loss": 0.0007,
+      "num_tokens": 933680.0,
+      "reward": 0.724609375,
+      "reward_std": 0.015291033312678337,
+      "rewards//mean": 0.724609375,
+      "rewards//std": 0.03565902262926102,
+      "step": 108
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0218,
+      "grad_norm": 2.94159197807312,
+      "kl": 0.01212124724406749,
+      "learning_rate": 9.996612835390594e-07,
+      "loss": 0.0005,
+      "num_tokens": 942360.0,
+      "reward": 0.712890625,
+      "reward_std": 0.011505991220474243,
+      "rewards//mean": 0.712890625,
+      "rewards//std": 0.0495036356151104,
+      "step": 109
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.022,
+      "grad_norm": 3.0552749633789062,
+      "kl": 0.018995069694938138,
+      "learning_rate": 9.996495043490283e-07,
+      "loss": 0.0008,
+      "num_tokens": 951000.0,
+      "reward": 0.74951171875,
+      "reward_std": 0.012679621577262878,
+      "rewards//mean": 0.74951171875,
+      "rewards//std": 0.04841279238462448,
+      "step": 110
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0222,
+      "grad_norm": 2.890538215637207,
+      "kl": 0.017626553802983835,
+      "learning_rate": 9.996375239002368e-07,
+      "loss": 0.0007,
+      "num_tokens": 959688.0,
+      "reward": 0.68048095703125,
+      "reward_std": 0.012808963656425476,
+      "rewards//mean": 0.68048095703125,
+      "rewards//std": 0.06301399320363998,
+      "step": 111
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0224,
+      "grad_norm": 3.0831246376037598,
+      "kl": 0.01691323594423011,
+      "learning_rate": 9.996253421975102e-07,
+      "loss": 0.0007,
+      "num_tokens": 968352.0,
+      "reward": 0.72821044921875,
+      "reward_std": 0.013352443464100361,
+      "rewards//mean": 0.72821044921875,
+      "rewards//std": 0.047239430248737335,
+      "step": 112
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0226,
+      "grad_norm": 3.094242572784424,
+      "kl": 0.017311632866039872,
+      "learning_rate": 9.996129592457556e-07,
+      "loss": 0.0007,
+      "num_tokens": 976936.0,
+      "reward": 0.72747802734375,
+      "reward_std": 0.01294796634465456,
+      "rewards//mean": 0.72747802734375,
+      "rewards//std": 0.04731083661317825,
+      "step": 113
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0228,
+      "grad_norm": 3.1343140602111816,
+      "kl": 0.01906962221255526,
+      "learning_rate": 9.996003750499607e-07,
+      "loss": 0.0008,
+      "num_tokens": 985552.0,
+      "reward": 0.7279052734375,
+      "reward_std": 0.013005741871893406,
+      "rewards//mean": 0.7279052734375,
+      "rewards//std": 0.040284812450408936,
+      "step": 114
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.023,
+      "grad_norm": 2.9134888648986816,
+      "kl": 0.015599035657942295,
+      "learning_rate": 9.995875896151944e-07,
+      "loss": 0.0006,
+      "num_tokens": 994264.0,
+      "reward": 0.7315673828125,
+      "reward_std": 0.01569860428571701,
+      "rewards//mean": 0.7315673828125,
+      "rewards//std": 0.03527295961976051,
+      "step": 115
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0232,
+      "grad_norm": 2.9819822311401367,
+      "kl": 0.021106205589603633,
+      "learning_rate": 9.99574602946607e-07,
+      "loss": 0.0008,
+      "num_tokens": 1002832.0,
+      "reward": 0.72247314453125,
+      "reward_std": 0.014688769355416298,
+      "rewards//mean": 0.72247314453125,
+      "rewards//std": 0.03920132666826248,
+      "step": 116
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0234,
+      "grad_norm": 2.9172799587249756,
+      "kl": 0.019246674666646868,
+      "learning_rate": 9.99561415049429e-07,
+      "loss": 0.0008,
+      "num_tokens": 1011448.0,
+      "reward": 0.72149658203125,
+      "reward_std": 0.015304185450077057,
+      "rewards//mean": 0.72149658203125,
+      "rewards//std": 0.03445466607809067,
+      "step": 117
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0236,
+      "grad_norm": 2.9187514781951904,
+      "kl": 0.02077232871670276,
+      "learning_rate": 9.99548025928973e-07,
+      "loss": 0.0008,
+      "num_tokens": 1020104.0,
+      "reward": 0.71710205078125,
+      "reward_std": 0.015331927686929703,
+      "rewards//mean": 0.71710205078125,
+      "rewards//std": 0.03905509039759636,
+      "step": 118
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0238,
+      "grad_norm": 2.793119430541992,
+      "kl": 0.01499595059431158,
+      "learning_rate": 9.995344355906318e-07,
+      "loss": 0.0006,
+      "num_tokens": 1028696.0,
+      "reward": 0.72967529296875,
+      "reward_std": 0.010924267582595348,
+      "rewards//mean": 0.72967529296875,
+      "rewards//std": 0.026722678914666176,
+      "step": 119
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.024,
+      "grad_norm": 2.850172519683838,
+      "kl": 0.016416702361311764,
+      "learning_rate": 9.995206440398796e-07,
+      "loss": 0.0007,
+      "num_tokens": 1037384.0,
+      "reward": 0.71246337890625,
+      "reward_std": 0.011186916381120682,
+      "rewards//mean": 0.71246337890625,
+      "rewards//std": 0.03915302827954292,
+      "step": 120
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0242,
+      "grad_norm": 3.1275367736816406,
+      "kl": 0.019138285715598613,
+      "learning_rate": 9.995066512822718e-07,
+      "loss": 0.0008,
+      "num_tokens": 1046176.0,
+      "reward": 0.7113037109375,
+      "reward_std": 0.012143395841121674,
+      "rewards//mean": 0.7113037109375,
+      "rewards//std": 0.040131207555532455,
+      "step": 121
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0244,
+      "grad_norm": 2.948559284210205,
+      "kl": 0.023869303578976542,
+      "learning_rate": 9.994924573234446e-07,
+      "loss": 0.001,
+      "num_tokens": 1054816.0,
+      "reward": 0.72149658203125,
+      "reward_std": 0.01686747930943966,
+      "rewards//mean": 0.72149658203125,
+      "rewards//std": 0.05836670845746994,
+      "step": 122
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0246,
+      "grad_norm": 2.984609603881836,
+      "kl": 0.021968798915622756,
+      "learning_rate": 9.994780621691154e-07,
+      "loss": 0.0009,
+      "num_tokens": 1063496.0,
+      "reward": 0.7352294921875,
+      "reward_std": 0.01451707910746336,
+      "rewards//mean": 0.7352294921875,
+      "rewards//std": 0.041131168603897095,
+      "step": 123
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0248,
+      "grad_norm": 2.9818739891052246,
+      "kl": 0.024137288215570152,
+      "learning_rate": 9.994634658250824e-07,
+      "loss": 0.001,
+      "num_tokens": 1072104.0,
+      "reward": 0.74267578125,
+      "reward_std": 0.012883680872619152,
+      "rewards//mean": 0.74267578125,
+      "rewards//std": 0.03866676986217499,
+      "step": 124
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.025,
+      "grad_norm": 2.7461442947387695,
+      "kl": 0.0202828252222389,
+      "learning_rate": 9.994486682972252e-07,
+      "loss": 0.0008,
+      "num_tokens": 1080752.0,
+      "reward": 0.7342529296875,
+      "reward_std": 0.013745720498263836,
+      "rewards//mean": 0.7342529296875,
+      "rewards//std": 0.0387197807431221,
+      "step": 125
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0252,
+      "grad_norm": 2.9171698093414307,
+      "kl": 0.022235158481635153,
+      "learning_rate": 9.99433669591504e-07,
+      "loss": 0.0009,
+      "num_tokens": 1089368.0,
+      "reward": 0.72381591796875,
+      "reward_std": 0.012552469968795776,
+      "rewards//mean": 0.72381591796875,
+      "rewards//std": 0.0521097406744957,
+      "step": 126
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0254,
+      "grad_norm": 2.6717565059661865,
+      "kl": 0.01895753329154104,
+      "learning_rate": 9.994184697139604e-07,
+      "loss": 0.0008,
+      "num_tokens": 1097992.0,
+      "reward": 0.73980712890625,
+      "reward_std": 0.011169027537107468,
+      "rewards//mean": 0.73980712890625,
+      "rewards//std": 0.026416730135679245,
+      "step": 127
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0256,
+      "grad_norm": 2.8336143493652344,
+      "kl": 0.02507422905182466,
+      "learning_rate": 9.99403068670717e-07,
+      "loss": 0.001,
+      "num_tokens": 1106576.0,
+      "reward": 0.7236328125,
+      "reward_std": 0.01609623059630394,
+      "rewards//mean": 0.7236328125,
+      "rewards//std": 0.04100455343723297,
+      "step": 128
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0258,
+      "grad_norm": 2.910475969314575,
+      "kl": 0.02157846326008439,
+      "learning_rate": 9.993874664679772e-07,
+      "loss": 0.0009,
+      "num_tokens": 1115160.0,
+      "reward": 0.71417236328125,
+      "reward_std": 0.012449681758880615,
+      "rewards//mean": 0.71417236328125,
+      "rewards//std": 0.044957876205444336,
+      "step": 129
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.026,
+      "grad_norm": 2.733341693878174,
+      "kl": 0.018211158807389438,
+      "learning_rate": 9.993716631120258e-07,
+      "loss": 0.0007,
+      "num_tokens": 1123808.0,
+      "reward": 0.7242431640625,
+      "reward_std": 0.01214703917503357,
+      "rewards//mean": 0.7242431640625,
+      "rewards//std": 0.0530952624976635,
+      "step": 130
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0262,
+      "grad_norm": 2.6225228309631348,
+      "kl": 0.029907271615229547,
+      "learning_rate": 9.99355658609228e-07,
+      "loss": 0.0012,
+      "num_tokens": 1132528.0,
+      "reward": 0.716552734375,
+      "reward_std": 0.01618252508342266,
+      "rewards//mean": 0.716552734375,
+      "rewards//std": 0.04933639243245125,
+      "step": 131
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0264,
+      "grad_norm": 2.5123965740203857,
+      "kl": 0.03157387347891927,
+      "learning_rate": 9.993394529660306e-07,
+      "loss": 0.0013,
+      "num_tokens": 1141160.0,
+      "reward": 0.73797607421875,
+      "reward_std": 0.012247065082192421,
+      "rewards//mean": 0.73797607421875,
+      "rewards//std": 0.038407862186431885,
+      "step": 132
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0266,
+      "grad_norm": 2.660794973373413,
+      "kl": 0.030565934779588133,
+      "learning_rate": 9.993230461889615e-07,
+      "loss": 0.0012,
+      "num_tokens": 1149744.0,
+      "reward": 0.72650146484375,
+      "reward_std": 0.012983234599232674,
+      "rewards//mean": 0.72650146484375,
+      "rewards//std": 0.0365544818341732,
+      "step": 133
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0268,
+      "grad_norm": 2.8524365425109863,
+      "kl": 0.028304936771746725,
+      "learning_rate": 9.993064382846289e-07,
+      "loss": 0.0011,
+      "num_tokens": 1158344.0,
+      "reward": 0.72186279296875,
+      "reward_std": 0.014238173142075539,
+      "rewards//mean": 0.72186279296875,
+      "rewards//std": 0.04674139618873596,
+      "step": 134
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.027,
+      "grad_norm": 2.772726535797119,
+      "kl": 0.028303387050982565,
+      "learning_rate": 9.992896292597228e-07,
+      "loss": 0.0011,
+      "num_tokens": 1166920.0,
+      "reward": 0.69146728515625,
+      "reward_std": 0.01865844428539276,
+      "rewards//mean": 0.69146728515625,
+      "rewards//std": 0.03763829916715622,
+      "step": 135
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0272,
+      "grad_norm": 2.759188652038574,
+      "kl": 0.027150681242346764,
+      "learning_rate": 9.992726191210137e-07,
+      "loss": 0.0011,
+      "num_tokens": 1175528.0,
+      "reward": 0.7373046875,
+      "reward_std": 0.011674551293253899,
+      "rewards//mean": 0.7373046875,
+      "rewards//std": 0.039217203855514526,
+      "step": 136
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0274,
+      "grad_norm": 2.642434597015381,
+      "kl": 0.027422477258369327,
+      "learning_rate": 9.992554078753533e-07,
+      "loss": 0.0011,
+      "num_tokens": 1184144.0,
+      "reward": 0.7293701171875,
+      "reward_std": 0.01614459417760372,
+      "rewards//mean": 0.7293701171875,
+      "rewards//std": 0.05587686598300934,
+      "step": 137
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0276,
+      "grad_norm": 2.7088518142700195,
+      "kl": 0.02350567257963121,
+      "learning_rate": 9.992379955296745e-07,
+      "loss": 0.0009,
+      "num_tokens": 1192832.0,
+      "reward": 0.73870849609375,
+      "reward_std": 0.01621604897081852,
+      "rewards//mean": 0.73870849609375,
+      "rewards//std": 0.044315218925476074,
+      "step": 138
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0278,
+      "grad_norm": 2.5840044021606445,
+      "kl": 0.03757087094709277,
+      "learning_rate": 9.992203820909905e-07,
+      "loss": 0.0015,
+      "num_tokens": 1201472.0,
+      "reward": 0.71124267578125,
+      "reward_std": 0.012270974926650524,
+      "rewards//mean": 0.71124267578125,
+      "rewards//std": 0.05214052274823189,
+      "step": 139
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.028,
+      "grad_norm": 2.7272589206695557,
+      "kl": 0.03142879402730614,
+      "learning_rate": 9.992025675663965e-07,
+      "loss": 0.0013,
+      "num_tokens": 1210176.0,
+      "reward": 0.7528076171875,
+      "reward_std": 0.012411152943968773,
+      "rewards//mean": 0.7528076171875,
+      "rewards//std": 0.03444615378975868,
+      "step": 140
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0282,
+      "grad_norm": 2.7319650650024414,
+      "kl": 0.028858942445367575,
+      "learning_rate": 9.991845519630676e-07,
+      "loss": 0.0012,
+      "num_tokens": 1218872.0,
+      "reward": 0.7288818359375,
+      "reward_std": 0.012398846447467804,
+      "rewards//mean": 0.7288818359375,
+      "rewards//std": 0.03197569400072098,
+      "step": 141
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0284,
+      "grad_norm": 3.004795789718628,
+      "kl": 0.0333898447570391,
+      "learning_rate": 9.991663352882613e-07,
+      "loss": 0.0013,
+      "num_tokens": 1227624.0,
+      "reward": 0.73577880859375,
+      "reward_std": 0.01382610946893692,
+      "rewards//mean": 0.73577880859375,
+      "rewards//std": 0.041894879192113876,
+      "step": 142
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0286,
+      "grad_norm": 2.6800928115844727,
+      "kl": 0.03263287292793393,
+      "learning_rate": 9.991479175493148e-07,
+      "loss": 0.0013,
+      "num_tokens": 1236264.0,
+      "reward": 0.74127197265625,
+      "reward_std": 0.012902824208140373,
+      "rewards//mean": 0.74127197265625,
+      "rewards//std": 0.036892492324113846,
+      "step": 143
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0288,
+      "grad_norm": 2.9141860008239746,
+      "kl": 0.027082599932327867,
+      "learning_rate": 9.991292987536468e-07,
+      "loss": 0.0011,
+      "num_tokens": 1244984.0,
+      "reward": 0.71527099609375,
+      "reward_std": 0.01237812452018261,
+      "rewards//mean": 0.71527099609375,
+      "rewards//std": 0.06423194706439972,
+      "step": 144
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.029,
+      "grad_norm": 2.536769151687622,
+      "kl": 0.03280840802472085,
+      "learning_rate": 9.991104789087569e-07,
+      "loss": 0.0013,
+      "num_tokens": 1253544.0,
+      "reward": 0.69854736328125,
+      "reward_std": 0.015090405941009521,
+      "rewards//mean": 0.69854736328125,
+      "rewards//std": 0.0490887388586998,
+      "step": 145
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0292,
+      "grad_norm": 2.6251041889190674,
+      "kl": 0.03762986068613827,
+      "learning_rate": 9.990914580222255e-07,
+      "loss": 0.0015,
+      "num_tokens": 1262272.0,
+      "reward": 0.75982666015625,
+      "reward_std": 0.013205408118665218,
+      "rewards//mean": 0.75982666015625,
+      "rewards//std": 0.04166662320494652,
+      "step": 146
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0294,
+      "grad_norm": 2.7780847549438477,
+      "kl": 0.029165279876906425,
+      "learning_rate": 9.990722361017149e-07,
+      "loss": 0.0012,
+      "num_tokens": 1270984.0,
+      "reward": 0.737548828125,
+      "reward_std": 0.013255832716822624,
+      "rewards//mean": 0.737548828125,
+      "rewards//std": 0.04294831305742264,
+      "step": 147
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0296,
+      "grad_norm": 2.728917360305786,
+      "kl": 0.03120983554981649,
+      "learning_rate": 9.990528131549671e-07,
+      "loss": 0.0012,
+      "num_tokens": 1279664.0,
+      "reward": 0.726806640625,
+      "reward_std": 0.016245994716882706,
+      "rewards//mean": 0.726806640625,
+      "rewards//std": 0.04418252781033516,
+      "step": 148
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0298,
+      "grad_norm": 2.602877140045166,
+      "kl": 0.03489436558447778,
+      "learning_rate": 9.990331891898058e-07,
+      "loss": 0.0014,
+      "num_tokens": 1288360.0,
+      "reward": 0.72125244140625,
+      "reward_std": 0.015676885843276978,
+      "rewards//mean": 0.72125244140625,
+      "rewards//std": 0.04169785603880882,
+      "step": 149
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.03,
+      "grad_norm": 2.4549598693847656,
+      "kl": 0.030647181847598404,
+      "learning_rate": 9.990133642141357e-07,
+      "loss": 0.0012,
+      "num_tokens": 1297032.0,
+      "reward": 0.74334716796875,
+      "reward_std": 0.013695335015654564,
+      "rewards//mean": 0.74334716796875,
+      "rewards//std": 0.03441246226429939,
+      "step": 150
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0302,
+      "grad_norm": 2.680105209350586,
+      "kl": 0.03840164153371006,
+      "learning_rate": 9.989933382359422e-07,
+      "loss": 0.0015,
+      "num_tokens": 1305632.0,
+      "reward": 0.71258544921875,
+      "reward_std": 0.012445718050003052,
+      "rewards//mean": 0.71258544921875,
+      "rewards//std": 0.04514201357960701,
+      "step": 151
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0304,
+      "grad_norm": 2.376408100128174,
+      "kl": 0.03482830408029258,
+      "learning_rate": 9.989731112632916e-07,
+      "loss": 0.0014,
+      "num_tokens": 1314272.0,
+      "reward": 0.72320556640625,
+      "reward_std": 0.009584907442331314,
+      "rewards//mean": 0.72320556640625,
+      "rewards//std": 0.04050455242395401,
+      "step": 152
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0306,
+      "grad_norm": 2.7264437675476074,
+      "kl": 0.038398853910621256,
+      "learning_rate": 9.989526833043316e-07,
+      "loss": 0.0015,
+      "num_tokens": 1322960.0,
+      "reward": 0.76617431640625,
+      "reward_std": 0.01662430167198181,
+      "rewards//mean": 0.76617431640625,
+      "rewards//std": 0.05045284330844879,
+      "step": 153
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0308,
+      "grad_norm": 2.79830002784729,
+      "kl": 0.045701892115175724,
+      "learning_rate": 9.989320543672903e-07,
+      "loss": 0.0018,
+      "num_tokens": 1331608.0,
+      "reward": 0.7325439453125,
+      "reward_std": 0.015923475846648216,
+      "rewards//mean": 0.7325439453125,
+      "rewards//std": 0.051498621702194214,
+      "step": 154
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.031,
+      "grad_norm": 2.336735486984253,
+      "kl": 0.0385560302529484,
+      "learning_rate": 9.989112244604771e-07,
+      "loss": 0.0015,
+      "num_tokens": 1340352.0,
+      "reward": 0.7415771484375,
+      "reward_std": 0.010023964568972588,
+      "rewards//mean": 0.7415771484375,
+      "rewards//std": 0.04613660275936127,
+      "step": 155
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0312,
+      "grad_norm": 2.459691286087036,
+      "kl": 0.02748531079851091,
+      "learning_rate": 9.988901935922825e-07,
+      "loss": 0.0011,
+      "num_tokens": 1349024.0,
+      "reward": 0.72869873046875,
+      "reward_std": 0.011690299957990646,
+      "rewards//mean": 0.72869873046875,
+      "rewards//std": 0.032983966171741486,
+      "step": 156
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0314,
+      "grad_norm": 2.55267333984375,
+      "kl": 0.03610196016961709,
+      "learning_rate": 9.988689617711776e-07,
+      "loss": 0.0014,
+      "num_tokens": 1357544.0,
+      "reward": 0.73175048828125,
+      "reward_std": 0.014921002089977264,
+      "rewards//mean": 0.73175048828125,
+      "rewards//std": 0.043061964213848114,
+      "step": 157
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0316,
+      "grad_norm": 2.3918230533599854,
+      "kl": 0.042030082491692156,
+      "learning_rate": 9.988475290057143e-07,
+      "loss": 0.0017,
+      "num_tokens": 1366224.0,
+      "reward": 0.73443603515625,
+      "reward_std": 0.011331222020089626,
+      "rewards//mean": 0.73443603515625,
+      "rewards//std": 0.04396059736609459,
+      "step": 158
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0318,
+      "grad_norm": 2.446019172668457,
+      "kl": 0.040930149145424366,
+      "learning_rate": 9.988258953045262e-07,
+      "loss": 0.0016,
+      "num_tokens": 1374848.0,
+      "reward": 0.73486328125,
+      "reward_std": 0.012925570830702782,
+      "rewards//mean": 0.73486328125,
+      "rewards//std": 0.043171197175979614,
+      "step": 159
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.032,
+      "grad_norm": 2.6987080574035645,
+      "kl": 0.036448456230573356,
+      "learning_rate": 9.988040606763272e-07,
+      "loss": 0.0015,
+      "num_tokens": 1383456.0,
+      "reward": 0.7001953125,
+      "reward_std": 0.010226636193692684,
+      "rewards//mean": 0.7001953125,
+      "rewards//std": 0.05315125733613968,
+      "step": 160
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0322,
+      "grad_norm": 2.816415309906006,
+      "kl": 0.046790738590061665,
+      "learning_rate": 9.98782025129912e-07,
+      "loss": 0.0019,
+      "num_tokens": 1392112.0,
+      "reward": 0.73614501953125,
+      "reward_std": 0.01567969098687172,
+      "rewards//mean": 0.73614501953125,
+      "rewards//std": 0.04891760274767876,
+      "step": 161
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0324,
+      "grad_norm": 2.4799399375915527,
+      "kl": 0.050143138971179724,
+      "learning_rate": 9.987597886741568e-07,
+      "loss": 0.002,
+      "num_tokens": 1400784.0,
+      "reward": 0.7564697265625,
+      "reward_std": 0.012243933975696564,
+      "rewards//mean": 0.7564697265625,
+      "rewards//std": 0.045272815972566605,
+      "step": 162
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0326,
+      "grad_norm": 2.50376558303833,
+      "kl": 0.041093296254985034,
+      "learning_rate": 9.987373513180184e-07,
+      "loss": 0.0016,
+      "num_tokens": 1409344.0,
+      "reward": 0.74749755859375,
+      "reward_std": 0.010359032079577446,
+      "rewards//mean": 0.74749755859375,
+      "rewards//std": 0.028775472193956375,
+      "step": 163
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0328,
+      "grad_norm": 2.3595211505889893,
+      "kl": 0.04736456251703203,
+      "learning_rate": 9.987147130705347e-07,
+      "loss": 0.0019,
+      "num_tokens": 1417920.0,
+      "reward": 0.7374267578125,
+      "reward_std": 0.01212608814239502,
+      "rewards//mean": 0.7374267578125,
+      "rewards//std": 0.039145924150943756,
+      "step": 164
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.033,
+      "grad_norm": 2.5590295791625977,
+      "kl": 0.04534601035993546,
+      "learning_rate": 9.98691873940824e-07,
+      "loss": 0.0018,
+      "num_tokens": 1426608.0,
+      "reward": 0.72821044921875,
+      "reward_std": 0.011726150289177895,
+      "rewards//mean": 0.72821044921875,
+      "rewards//std": 0.045850057154893875,
+      "step": 165
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0332,
+      "grad_norm": 2.226454019546509,
+      "kl": 0.047674126049969345,
+      "learning_rate": 9.98668833938086e-07,
+      "loss": 0.0019,
+      "num_tokens": 1435216.0,
+      "reward": 0.72979736328125,
+      "reward_std": 0.010330487042665482,
+      "rewards//mean": 0.72979736328125,
+      "rewards//std": 0.03575774282217026,
+      "step": 166
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0334,
+      "grad_norm": 2.772519588470459,
+      "kl": 0.03843457077164203,
+      "learning_rate": 9.986455930716016e-07,
+      "loss": 0.0015,
+      "num_tokens": 1443832.0,
+      "reward": 0.69500732421875,
+      "reward_std": 0.01148445438593626,
+      "rewards//mean": 0.69500732421875,
+      "rewards//std": 0.059181105345487595,
+      "step": 167
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0336,
+      "grad_norm": 2.318955183029175,
+      "kl": 0.05194610939361155,
+      "learning_rate": 9.986221513507318e-07,
+      "loss": 0.0021,
+      "num_tokens": 1452488.0,
+      "reward": 0.75311279296875,
+      "reward_std": 0.011434704065322876,
+      "rewards//mean": 0.75311279296875,
+      "rewards//std": 0.03438781946897507,
+      "step": 168
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0338,
+      "grad_norm": 2.503162145614624,
+      "kl": 0.043565331608988345,
+      "learning_rate": 9.985985087849191e-07,
+      "loss": 0.0017,
+      "num_tokens": 1461184.0,
+      "reward": 0.73974609375,
+      "reward_std": 0.011233117431402206,
+      "rewards//mean": 0.73974609375,
+      "rewards//std": 0.042623523622751236,
+      "step": 169
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.034,
+      "grad_norm": 2.350789785385132,
+      "kl": 0.03935895999893546,
+      "learning_rate": 9.985746653836866e-07,
+      "loss": 0.0016,
+      "num_tokens": 1469920.0,
+      "reward": 0.7506103515625,
+      "reward_std": 0.011013064533472061,
+      "rewards//mean": 0.7506103515625,
+      "rewards//std": 0.044315777719020844,
+      "step": 170
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0342,
+      "grad_norm": 2.4444878101348877,
+      "kl": 0.04686792800202966,
+      "learning_rate": 9.985506211566386e-07,
+      "loss": 0.0019,
+      "num_tokens": 1478560.0,
+      "reward": 0.72991943359375,
+      "reward_std": 0.014785466715693474,
+      "rewards//mean": 0.72991943359375,
+      "rewards//std": 0.04804300516843796,
+      "step": 171
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0344,
+      "grad_norm": 2.1867735385894775,
+      "kl": 0.04372467682696879,
+      "learning_rate": 9.9852637611346e-07,
+      "loss": 0.0017,
+      "num_tokens": 1487232.0,
+      "reward": 0.6925048828125,
+      "reward_std": 0.011685654520988464,
+      "rewards//mean": 0.6925048828125,
+      "rewards//std": 0.0518384724855423,
+      "step": 172
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0346,
+      "grad_norm": 2.351771354675293,
+      "kl": 0.052237617783248425,
+      "learning_rate": 9.98501930263917e-07,
+      "loss": 0.0021,
+      "num_tokens": 1495848.0,
+      "reward": 0.720947265625,
+      "reward_std": 0.012654958292841911,
+      "rewards//mean": 0.720947265625,
+      "rewards//std": 0.04899648576974869,
+      "step": 173
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0348,
+      "grad_norm": 2.2505569458007812,
+      "kl": 0.050110013922676444,
+      "learning_rate": 9.984772836178556e-07,
+      "loss": 0.002,
+      "num_tokens": 1504680.0,
+      "reward": 0.74395751953125,
+      "reward_std": 0.012244937010109425,
+      "rewards//mean": 0.74395751953125,
+      "rewards//std": 0.04974732547998428,
+      "step": 174
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.035,
+      "grad_norm": 2.456139087677002,
+      "kl": 0.0581817498896271,
+      "learning_rate": 9.984524361852043e-07,
+      "loss": 0.0023,
+      "num_tokens": 1513360.0,
+      "reward": 0.71875,
+      "reward_std": 0.009412692859768867,
+      "rewards//mean": 0.71875,
+      "rewards//std": 0.05139990895986557,
+      "step": 175
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0352,
+      "grad_norm": 2.341621160507202,
+      "kl": 0.050122945569455624,
+      "learning_rate": 9.984273879759712e-07,
+      "loss": 0.002,
+      "num_tokens": 1522112.0,
+      "reward": 0.7413330078125,
+      "reward_std": 0.010432751849293709,
+      "rewards//mean": 0.7413330078125,
+      "rewards//std": 0.0366072952747345,
+      "step": 176
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0354,
+      "grad_norm": 2.1102633476257324,
+      "kl": 0.050346111645922065,
+      "learning_rate": 9.984021390002457e-07,
+      "loss": 0.002,
+      "num_tokens": 1530848.0,
+      "reward": 0.71044921875,
+      "reward_std": 0.010294707491993904,
+      "rewards//mean": 0.71044921875,
+      "rewards//std": 0.04748334363102913,
+      "step": 177
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0356,
+      "grad_norm": 2.223541736602783,
+      "kl": 0.048459484125487506,
+      "learning_rate": 9.983766892681985e-07,
+      "loss": 0.0019,
+      "num_tokens": 1539528.0,
+      "reward": 0.72589111328125,
+      "reward_std": 0.012135764583945274,
+      "rewards//mean": 0.72589111328125,
+      "rewards//std": 0.03706156834959984,
+      "step": 178
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0358,
+      "grad_norm": 2.265479326248169,
+      "kl": 0.04988991259597242,
+      "learning_rate": 9.983510387900802e-07,
+      "loss": 0.002,
+      "num_tokens": 1548192.0,
+      "reward": 0.70721435546875,
+      "reward_std": 0.011628516018390656,
+      "rewards//mean": 0.70721435546875,
+      "rewards//std": 0.05118945613503456,
+      "step": 179
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.036,
+      "grad_norm": 2.2445435523986816,
+      "kl": 0.05180193926207721,
+      "learning_rate": 9.983251875762232e-07,
+      "loss": 0.0021,
+      "num_tokens": 1556856.0,
+      "reward": 0.73931884765625,
+      "reward_std": 0.011930052191019058,
+      "rewards//mean": 0.73931884765625,
+      "rewards//std": 0.049146369099617004,
+      "step": 180
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0362,
+      "grad_norm": 1.988173007965088,
+      "kl": 0.04608521016780287,
+      "learning_rate": 9.982991356370403e-07,
+      "loss": 0.0018,
+      "num_tokens": 1565488.0,
+      "reward": 0.7384033203125,
+      "reward_std": 0.012451891787350178,
+      "rewards//mean": 0.7384033203125,
+      "rewards//std": 0.053000520914793015,
+      "step": 181
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0364,
+      "grad_norm": 2.1265347003936768,
+      "kl": 0.05836852011270821,
+      "learning_rate": 9.98272882983025e-07,
+      "loss": 0.0023,
+      "num_tokens": 1574184.0,
+      "reward": 0.7396240234375,
+      "reward_std": 0.012791775166988373,
+      "rewards//mean": 0.7396240234375,
+      "rewards//std": 0.0466274619102478,
+      "step": 182
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0366,
+      "grad_norm": 2.3052797317504883,
+      "kl": 0.05570454359985888,
+      "learning_rate": 9.982464296247522e-07,
+      "loss": 0.0022,
+      "num_tokens": 1582888.0,
+      "reward": 0.737548828125,
+      "reward_std": 0.011044729501008987,
+      "rewards//mean": 0.737548828125,
+      "rewards//std": 0.03410276025533676,
+      "step": 183
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0368,
+      "grad_norm": 2.3991541862487793,
+      "kl": 0.04946473188465461,
+      "learning_rate": 9.98219775572877e-07,
+      "loss": 0.002,
+      "num_tokens": 1591408.0,
+      "reward": 0.754150390625,
+      "reward_std": 0.010761473327875137,
+      "rewards//mean": 0.754150390625,
+      "rewards//std": 0.03539912402629852,
+      "step": 184
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.037,
+      "grad_norm": 2.1395928859710693,
+      "kl": 0.055561707355082035,
+      "learning_rate": 9.981929208381357e-07,
+      "loss": 0.0022,
+      "num_tokens": 1600088.0,
+      "reward": 0.767333984375,
+      "reward_std": 0.012706330046057701,
+      "rewards//mean": 0.767333984375,
+      "rewards//std": 0.03725268691778183,
+      "step": 185
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0372,
+      "grad_norm": 2.055741786956787,
+      "kl": 0.05260586831718683,
+      "learning_rate": 9.981658654313456e-07,
+      "loss": 0.0021,
+      "num_tokens": 1608712.0,
+      "reward": 0.72021484375,
+      "reward_std": 0.00924680382013321,
+      "rewards//mean": 0.72021484375,
+      "rewards//std": 0.038509856909513474,
+      "step": 186
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0374,
+      "grad_norm": 2.327390432357788,
+      "kl": 0.054784362437203526,
+      "learning_rate": 9.981386093634045e-07,
+      "loss": 0.0022,
+      "num_tokens": 1617400.0,
+      "reward": 0.7587890625,
+      "reward_std": 0.010368096642196178,
+      "rewards//mean": 0.7587890625,
+      "rewards//std": 0.02377607673406601,
+      "step": 187
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0376,
+      "grad_norm": 2.1157243251800537,
+      "kl": 0.059309986187145114,
+      "learning_rate": 9.98111152645291e-07,
+      "loss": 0.0024,
+      "num_tokens": 1625992.0,
+      "reward": 0.7705078125,
+      "reward_std": 0.01089094765484333,
+      "rewards//mean": 0.7705078125,
+      "rewards//std": 0.040253423154354095,
+      "step": 188
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0378,
+      "grad_norm": 2.4641757011413574,
+      "kl": 0.05123629025183618,
+      "learning_rate": 9.98083495288065e-07,
+      "loss": 0.002,
+      "num_tokens": 1634576.0,
+      "reward": 0.71209716796875,
+      "reward_std": 0.015146300196647644,
+      "rewards//mean": 0.71209716796875,
+      "rewards//std": 0.04571184143424034,
+      "step": 189
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.038,
+      "grad_norm": 2.296196699142456,
+      "kl": 0.05166075681336224,
+      "learning_rate": 9.980556373028665e-07,
+      "loss": 0.0021,
+      "num_tokens": 1643200.0,
+      "reward": 0.73907470703125,
+      "reward_std": 0.008769819512963295,
+      "rewards//mean": 0.73907470703125,
+      "rewards//std": 0.03626636788249016,
+      "step": 190
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0382,
+      "grad_norm": 2.1193857192993164,
+      "kl": 0.05269998754374683,
+      "learning_rate": 9.98027578700917e-07,
+      "loss": 0.0021,
+      "num_tokens": 1651848.0,
+      "reward": 0.70281982421875,
+      "reward_std": 0.010136201977729797,
+      "rewards//mean": 0.70281982421875,
+      "rewards//std": 0.03540205955505371,
+      "step": 191
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0384,
+      "grad_norm": 2.1637582778930664,
+      "kl": 0.051795421401038766,
+      "learning_rate": 9.979993194935182e-07,
+      "loss": 0.0021,
+      "num_tokens": 1660472.0,
+      "reward": 0.739013671875,
+      "reward_std": 0.01057550311088562,
+      "rewards//mean": 0.739013671875,
+      "rewards//std": 0.04138597846031189,
+      "step": 192
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0386,
+      "grad_norm": 2.1885793209075928,
+      "kl": 0.05414393660612404,
+      "learning_rate": 9.979708596920529e-07,
+      "loss": 0.0022,
+      "num_tokens": 1669128.0,
+      "reward": 0.7567138671875,
+      "reward_std": 0.009756932035088539,
+      "rewards//mean": 0.7567138671875,
+      "rewards//std": 0.029720351099967957,
+      "step": 193
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0388,
+      "grad_norm": 2.267918109893799,
+      "kl": 0.05917287478223443,
+      "learning_rate": 9.97942199307985e-07,
+      "loss": 0.0024,
+      "num_tokens": 1677784.0,
+      "reward": 0.76678466796875,
+      "reward_std": 0.013261063024401665,
+      "rewards//mean": 0.76678466796875,
+      "rewards//std": 0.047738611698150635,
+      "step": 194
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.039,
+      "grad_norm": 2.1623001098632812,
+      "kl": 0.05870238086208701,
+      "learning_rate": 9.97913338352859e-07,
+      "loss": 0.0023,
+      "num_tokens": 1686448.0,
+      "reward": 0.73870849609375,
+      "reward_std": 0.008617502637207508,
+      "rewards//mean": 0.73870849609375,
+      "rewards//std": 0.026070643216371536,
+      "step": 195
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0392,
+      "grad_norm": 2.185559034347534,
+      "kl": 0.05918487114831805,
+      "learning_rate": 9.978842768382998e-07,
+      "loss": 0.0024,
+      "num_tokens": 1695072.0,
+      "reward": 0.7078857421875,
+      "reward_std": 0.011204719543457031,
+      "rewards//mean": 0.7078857421875,
+      "rewards//std": 0.02731688879430294,
+      "step": 196
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0394,
+      "grad_norm": 2.2516632080078125,
+      "kl": 0.054595754481852055,
+      "learning_rate": 9.978550147760131e-07,
+      "loss": 0.0022,
+      "num_tokens": 1703680.0,
+      "reward": 0.74151611328125,
+      "reward_std": 0.01452554203569889,
+      "rewards//mean": 0.74151611328125,
+      "rewards//std": 0.04260678216814995,
+      "step": 197
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0396,
+      "grad_norm": 2.036886215209961,
+      "kl": 0.05572623899206519,
+      "learning_rate": 9.978255521777862e-07,
+      "loss": 0.0022,
+      "num_tokens": 1712304.0,
+      "reward": 0.734130859375,
+      "reward_std": 0.008719809353351593,
+      "rewards//mean": 0.734130859375,
+      "rewards//std": 0.043896544724702835,
+      "step": 198
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0398,
+      "grad_norm": 2.373351573944092,
+      "kl": 0.0579618860501796,
+      "learning_rate": 9.977958890554866e-07,
+      "loss": 0.0023,
+      "num_tokens": 1720936.0,
+      "reward": 0.71710205078125,
+      "reward_std": 0.01314060389995575,
+      "rewards//mean": 0.71710205078125,
+      "rewards//std": 0.034784361720085144,
+      "step": 199
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.04,
+      "grad_norm": 3.3971798419952393,
+      "kl": 0.05702702864073217,
+      "learning_rate": 9.97766025421062e-07,
+      "loss": 0.0023,
+      "num_tokens": 1729552.0,
+      "reward": 0.71942138671875,
+      "reward_std": 0.014108622446656227,
+      "rewards//mean": 0.71942138671875,
+      "rewards//std": 0.03779282793402672,
+      "step": 200
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0402,
+      "grad_norm": 1.8567906618118286,
+      "kl": 0.05714975926093757,
+      "learning_rate": 9.977359612865422e-07,
+      "loss": 0.0023,
+      "num_tokens": 1738184.0,
+      "reward": 0.7197265625,
+      "reward_std": 0.012525375932455063,
+      "rewards//mean": 0.7197265625,
+      "rewards//std": 0.035187240689992905,
+      "step": 201
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0404,
+      "grad_norm": 2.1849992275238037,
+      "kl": 0.061090546660125256,
+      "learning_rate": 9.977056966640367e-07,
+      "loss": 0.0024,
+      "num_tokens": 1746792.0,
+      "reward": 0.736572265625,
+      "reward_std": 0.016942109912633896,
+      "rewards//mean": 0.736572265625,
+      "rewards//std": 0.040252674371004105,
+      "step": 202
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0406,
+      "grad_norm": 2.810636281967163,
+      "kl": 0.055643110536038876,
+      "learning_rate": 9.976752315657359e-07,
+      "loss": 0.0022,
+      "num_tokens": 1755408.0,
+      "reward": 0.74658203125,
+      "reward_std": 0.013593493029475212,
+      "rewards//mean": 0.74658203125,
+      "rewards//std": 0.039881665259599686,
+      "step": 203
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0408,
+      "grad_norm": 2.388843059539795,
+      "kl": 0.06320055993273854,
+      "learning_rate": 9.976445660039117e-07,
+      "loss": 0.0025,
+      "num_tokens": 1764008.0,
+      "reward": 0.7498779296875,
+      "reward_std": 0.013801433145999908,
+      "rewards//mean": 0.7498779296875,
+      "rewards//std": 0.03442857041954994,
+      "step": 204
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.041,
+      "grad_norm": 2.3890225887298584,
+      "kl": 0.06138871354050934,
+      "learning_rate": 9.976136999909155e-07,
+      "loss": 0.0025,
+      "num_tokens": 1772688.0,
+      "reward": 0.73956298828125,
+      "reward_std": 0.010558126494288445,
+      "rewards//mean": 0.73956298828125,
+      "rewards//std": 0.045245181769132614,
+      "step": 205
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0412,
+      "grad_norm": 2.236191511154175,
+      "kl": 0.06445245840586722,
+      "learning_rate": 9.975826335391805e-07,
+      "loss": 0.0026,
+      "num_tokens": 1781256.0,
+      "reward": 0.7501220703125,
+      "reward_std": 0.012905829586088657,
+      "rewards//mean": 0.7501220703125,
+      "rewards//std": 0.03487589955329895,
+      "step": 206
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0414,
+      "grad_norm": 2.0409834384918213,
+      "kl": 0.061787127051502466,
+      "learning_rate": 9.975513666612203e-07,
+      "loss": 0.0025,
+      "num_tokens": 1789976.0,
+      "reward": 0.74481201171875,
+      "reward_std": 0.011488648131489754,
+      "rewards//mean": 0.74481201171875,
+      "rewards//std": 0.05587073415517807,
+      "step": 207
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0416,
+      "grad_norm": 2.1245434284210205,
+      "kl": 0.05468311160802841,
+      "learning_rate": 9.975198993696291e-07,
+      "loss": 0.0022,
+      "num_tokens": 1798664.0,
+      "reward": 0.73492431640625,
+      "reward_std": 0.013937927782535553,
+      "rewards//mean": 0.73492431640625,
+      "rewards//std": 0.0415378138422966,
+      "step": 208
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0418,
+      "grad_norm": 2.1178219318389893,
+      "kl": 0.05903259618207812,
+      "learning_rate": 9.97488231677082e-07,
+      "loss": 0.0024,
+      "num_tokens": 1807424.0,
+      "reward": 0.68988037109375,
+      "reward_std": 0.01218400988727808,
+      "rewards//mean": 0.68988037109375,
+      "rewards//std": 0.04769261181354523,
+      "step": 209
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.042,
+      "grad_norm": 1.9095737934112549,
+      "kl": 0.05956404539756477,
+      "learning_rate": 9.974563635963347e-07,
+      "loss": 0.0024,
+      "num_tokens": 1816088.0,
+      "reward": 0.73046875,
+      "reward_std": 0.009744959883391857,
+      "rewards//mean": 0.73046875,
+      "rewards//std": 0.04048740491271019,
+      "step": 210
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0422,
+      "grad_norm": 2.1075732707977295,
+      "kl": 0.06340880901552737,
+      "learning_rate": 9.974242951402235e-07,
+      "loss": 0.0025,
+      "num_tokens": 1824672.0,
+      "reward": 0.6942138671875,
+      "reward_std": 0.00884406641125679,
+      "rewards//mean": 0.6942138671875,
+      "rewards//std": 0.04520455375313759,
+      "step": 211
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0424,
+      "grad_norm": 2.276707887649536,
+      "kl": 0.05456857895478606,
+      "learning_rate": 9.973920263216657e-07,
+      "loss": 0.0022,
+      "num_tokens": 1833248.0,
+      "reward": 0.7916259765625,
+      "reward_std": 0.011873690411448479,
+      "rewards//mean": 0.7916259765625,
+      "rewards//std": 0.027945812791585922,
+      "step": 212
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0426,
+      "grad_norm": 2.3808939456939697,
+      "kl": 0.061990561662241817,
+      "learning_rate": 9.97359557153659e-07,
+      "loss": 0.0025,
+      "num_tokens": 1841808.0,
+      "reward": 0.73394775390625,
+      "reward_std": 0.01026148721575737,
+      "rewards//mean": 0.73394775390625,
+      "rewards//std": 0.0428367480635643,
+      "step": 213
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0428,
+      "grad_norm": 2.4553043842315674,
+      "kl": 0.061710300971753895,
+      "learning_rate": 9.973268876492825e-07,
+      "loss": 0.0025,
+      "num_tokens": 1850392.0,
+      "reward": 0.73028564453125,
+      "reward_std": 0.015301035717129707,
+      "rewards//mean": 0.73028564453125,
+      "rewards//std": 0.04240093007683754,
+      "step": 214
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.043,
+      "grad_norm": 2.1479804515838623,
+      "kl": 0.06337157567031682,
+      "learning_rate": 9.972940178216952e-07,
+      "loss": 0.0025,
+      "num_tokens": 1859016.0,
+      "reward": 0.7620849609375,
+      "reward_std": 0.011465619318187237,
+      "rewards//mean": 0.7620849609375,
+      "rewards//std": 0.04265884682536125,
+      "step": 215
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0432,
+      "grad_norm": 2.276979923248291,
+      "kl": 0.060141903813928366,
+      "learning_rate": 9.972609476841365e-07,
+      "loss": 0.0024,
+      "num_tokens": 1867616.0,
+      "reward": 0.7412109375,
+      "reward_std": 0.008193857036530972,
+      "rewards//mean": 0.7412109375,
+      "rewards//std": 0.021133441478013992,
+      "step": 216
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0434,
+      "grad_norm": 2.192436933517456,
+      "kl": 0.06401014095172286,
+      "learning_rate": 9.97227677249928e-07,
+      "loss": 0.0026,
+      "num_tokens": 1876296.0,
+      "reward": 0.745361328125,
+      "reward_std": 0.010227463208138943,
+      "rewards//mean": 0.745361328125,
+      "rewards//std": 0.036384351551532745,
+      "step": 217
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0436,
+      "grad_norm": 2.2693440914154053,
+      "kl": 0.05632793949916959,
+      "learning_rate": 9.971942065324702e-07,
+      "loss": 0.0023,
+      "num_tokens": 1884904.0,
+      "reward": 0.7484130859375,
+      "reward_std": 0.013199035078287125,
+      "rewards//mean": 0.7484130859375,
+      "rewards//std": 0.03698413819074631,
+      "step": 218
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0438,
+      "grad_norm": 2.1643688678741455,
+      "kl": 0.055894225370138884,
+      "learning_rate": 9.971605355452457e-07,
+      "loss": 0.0022,
+      "num_tokens": 1893616.0,
+      "reward": 0.72784423828125,
+      "reward_std": 0.011329513974487782,
+      "rewards//mean": 0.72784423828125,
+      "rewards//std": 0.044684987515211105,
+      "step": 219
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.044,
+      "grad_norm": 2.3612186908721924,
+      "kl": 0.07208431093022227,
+      "learning_rate": 9.97126664301817e-07,
+      "loss": 0.0029,
+      "num_tokens": 1902160.0,
+      "reward": 0.7008056640625,
+      "reward_std": 0.014414534904062748,
+      "rewards//mean": 0.7008056640625,
+      "rewards//std": 0.04989214614033699,
+      "step": 220
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0442,
+      "grad_norm": 2.268941640853882,
+      "kl": 0.06697740312665701,
+      "learning_rate": 9.970925928158272e-07,
+      "loss": 0.0027,
+      "num_tokens": 1910880.0,
+      "reward": 0.73345947265625,
+      "reward_std": 0.012288164347410202,
+      "rewards//mean": 0.73345947265625,
+      "rewards//std": 0.03685308247804642,
+      "step": 221
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0444,
+      "grad_norm": 2.4280242919921875,
+      "kl": 0.06854389910586178,
+      "learning_rate": 9.970583211010007e-07,
+      "loss": 0.0027,
+      "num_tokens": 1919640.0,
+      "reward": 0.70794677734375,
+      "reward_std": 0.012792366556823254,
+      "rewards//mean": 0.70794677734375,
+      "rewards//std": 0.04355793073773384,
+      "step": 222
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0446,
+      "grad_norm": 2.040374994277954,
+      "kl": 0.05879002343863249,
+      "learning_rate": 9.970238491711415e-07,
+      "loss": 0.0024,
+      "num_tokens": 1928296.0,
+      "reward": 0.728271484375,
+      "reward_std": 0.00894878152757883,
+      "rewards//mean": 0.728271484375,
+      "rewards//std": 0.03154986351728439,
+      "step": 223
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0448,
+      "grad_norm": 2.0993146896362305,
+      "kl": 0.06133840745314956,
+      "learning_rate": 9.969891770401356e-07,
+      "loss": 0.0025,
+      "num_tokens": 1937088.0,
+      "reward": 0.75421142578125,
+      "reward_std": 0.010610558092594147,
+      "rewards//mean": 0.75421142578125,
+      "rewards//std": 0.03229665383696556,
+      "step": 224
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.045,
+      "grad_norm": 2.6484577655792236,
+      "kl": 0.0712521958630532,
+      "learning_rate": 9.969543047219486e-07,
+      "loss": 0.0029,
+      "num_tokens": 1945688.0,
+      "reward": 0.76324462890625,
+      "reward_std": 0.016112633049488068,
+      "rewards//mean": 0.76324462890625,
+      "rewards//std": 0.04176894947886467,
+      "step": 225
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0452,
+      "grad_norm": 2.331159830093384,
+      "kl": 0.07147391699254513,
+      "learning_rate": 9.96919232230627e-07,
+      "loss": 0.0029,
+      "num_tokens": 1954320.0,
+      "reward": 0.7493896484375,
+      "reward_std": 0.009902137331664562,
+      "rewards//mean": 0.7493896484375,
+      "rewards//std": 0.04099993780255318,
+      "step": 226
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0454,
+      "grad_norm": 2.0388107299804688,
+      "kl": 0.07578787580132484,
+      "learning_rate": 9.968839595802981e-07,
+      "loss": 0.003,
+      "num_tokens": 1962944.0,
+      "reward": 0.71954345703125,
+      "reward_std": 0.010332188569009304,
+      "rewards//mean": 0.71954345703125,
+      "rewards//std": 0.0278586708009243,
+      "step": 227
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0456,
+      "grad_norm": 2.3314082622528076,
+      "kl": 0.06498332298360765,
+      "learning_rate": 9.968484867851697e-07,
+      "loss": 0.0026,
+      "num_tokens": 1971624.0,
+      "reward": 0.76806640625,
+      "reward_std": 0.013963586650788784,
+      "rewards//mean": 0.76806640625,
+      "rewards//std": 0.04706322029232979,
+      "step": 228
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0458,
+      "grad_norm": 2.018660545349121,
+      "kl": 0.07186876214109361,
+      "learning_rate": 9.968128138595302e-07,
+      "loss": 0.0029,
+      "num_tokens": 1980280.0,
+      "reward": 0.7147216796875,
+      "reward_std": 0.010772351175546646,
+      "rewards//mean": 0.7147216796875,
+      "rewards//std": 0.03680853918194771,
+      "step": 229
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.046,
+      "grad_norm": 2.121382236480713,
+      "kl": 0.06785159790888429,
+      "learning_rate": 9.967769408177488e-07,
+      "loss": 0.0027,
+      "num_tokens": 1988880.0,
+      "reward": 0.6982421875,
+      "reward_std": 0.012861795723438263,
+      "rewards//mean": 0.6982421875,
+      "rewards//std": 0.0487145259976387,
+      "step": 230
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0462,
+      "grad_norm": 2.435134172439575,
+      "kl": 0.08035403094254434,
+      "learning_rate": 9.967408676742751e-07,
+      "loss": 0.0032,
+      "num_tokens": 1997536.0,
+      "reward": 0.7432861328125,
+      "reward_std": 0.012507785111665726,
+      "rewards//mean": 0.7432861328125,
+      "rewards//std": 0.036315012723207474,
+      "step": 231
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0464,
+      "grad_norm": 2.347782611846924,
+      "kl": 0.07359315385110676,
+      "learning_rate": 9.967045944436393e-07,
+      "loss": 0.0029,
+      "num_tokens": 2006280.0,
+      "reward": 0.7220458984375,
+      "reward_std": 0.010751600377261639,
+      "rewards//mean": 0.7220458984375,
+      "rewards//std": 0.04129425063729286,
+      "step": 232
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0466,
+      "grad_norm": 2.1992995738983154,
+      "kl": 0.0674269802402705,
+      "learning_rate": 9.96668121140452e-07,
+      "loss": 0.0027,
+      "num_tokens": 2015040.0,
+      "reward": 0.7586669921875,
+      "reward_std": 0.014760918915271759,
+      "rewards//mean": 0.7586669921875,
+      "rewards//std": 0.03706100583076477,
+      "step": 233
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0468,
+      "grad_norm": 1.9348255395889282,
+      "kl": 0.07445071451365948,
+      "learning_rate": 9.966314477794052e-07,
+      "loss": 0.003,
+      "num_tokens": 2023640.0,
+      "reward": 0.73541259765625,
+      "reward_std": 0.009623780846595764,
+      "rewards//mean": 0.73541259765625,
+      "rewards//std": 0.03704645112156868,
+      "step": 234
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.047,
+      "grad_norm": 2.164318323135376,
+      "kl": 0.07863690797239542,
+      "learning_rate": 9.965945743752705e-07,
+      "loss": 0.0031,
+      "num_tokens": 2032216.0,
+      "reward": 0.7347412109375,
+      "reward_std": 0.010528791695833206,
+      "rewards//mean": 0.7347412109375,
+      "rewards//std": 0.041150301694869995,
+      "step": 235
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0472,
+      "grad_norm": 2.256127119064331,
+      "kl": 0.0846207330469042,
+      "learning_rate": 9.965575009429005e-07,
+      "loss": 0.0034,
+      "num_tokens": 2040856.0,
+      "reward": 0.7567138671875,
+      "reward_std": 0.012185719795525074,
+      "rewards//mean": 0.7567138671875,
+      "rewards//std": 0.038885194808244705,
+      "step": 236
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0474,
+      "grad_norm": 2.367684841156006,
+      "kl": 0.08532239287160337,
+      "learning_rate": 9.965202274972286e-07,
+      "loss": 0.0034,
+      "num_tokens": 2049408.0,
+      "reward": 0.71923828125,
+      "reward_std": 0.009885359555482864,
+      "rewards//mean": 0.71923828125,
+      "rewards//std": 0.034121397882699966,
+      "step": 237
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0476,
+      "grad_norm": 2.3444292545318604,
+      "kl": 0.07901590317487717,
+      "learning_rate": 9.964827540532684e-07,
+      "loss": 0.0032,
+      "num_tokens": 2058016.0,
+      "reward": 0.7315673828125,
+      "reward_std": 0.013179188594222069,
+      "rewards//mean": 0.7315673828125,
+      "rewards//std": 0.04617464542388916,
+      "step": 238
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0478,
+      "grad_norm": 2.395791530609131,
+      "kl": 0.08098797150887549,
+      "learning_rate": 9.964450806261144e-07,
+      "loss": 0.0032,
+      "num_tokens": 2066648.0,
+      "reward": 0.75616455078125,
+      "reward_std": 0.012934434227645397,
+      "rewards//mean": 0.75616455078125,
+      "rewards//std": 0.029723087325692177,
+      "step": 239
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.048,
+      "grad_norm": 2.1232640743255615,
+      "kl": 0.0727109950967133,
+      "learning_rate": 9.96407207230941e-07,
+      "loss": 0.0029,
+      "num_tokens": 2075336.0,
+      "reward": 0.7333984375,
+      "reward_std": 0.010355428792536259,
+      "rewards//mean": 0.7333984375,
+      "rewards//std": 0.03930357098579407,
+      "step": 240
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0482,
+      "grad_norm": 2.341886043548584,
+      "kl": 0.07593965763226151,
+      "learning_rate": 9.963691338830042e-07,
+      "loss": 0.003,
+      "num_tokens": 2083952.0,
+      "reward": 0.7322998046875,
+      "reward_std": 0.011291599832475185,
+      "rewards//mean": 0.7322998046875,
+      "rewards//std": 0.02264885976910591,
+      "step": 241
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0484,
+      "grad_norm": 2.3043925762176514,
+      "kl": 0.08260205248370767,
+      "learning_rate": 9.963308605976396e-07,
+      "loss": 0.0033,
+      "num_tokens": 2092624.0,
+      "reward": 0.76416015625,
+      "reward_std": 0.00958188809454441,
+      "rewards//mean": 0.76416015625,
+      "rewards//std": 0.02448371797800064,
+      "step": 242
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0486,
+      "grad_norm": 2.2695934772491455,
+      "kl": 0.07344825379550457,
+      "learning_rate": 9.962923873902636e-07,
+      "loss": 0.0029,
+      "num_tokens": 2101160.0,
+      "reward": 0.70928955078125,
+      "reward_std": 0.010659701190888882,
+      "rewards//mean": 0.70928955078125,
+      "rewards//std": 0.044932279735803604,
+      "step": 243
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0488,
+      "grad_norm": 2.400766611099243,
+      "kl": 0.08081559091806412,
+      "learning_rate": 9.962537142763732e-07,
+      "loss": 0.0032,
+      "num_tokens": 2109792.0,
+      "reward": 0.74383544921875,
+      "reward_std": 0.010848671197891235,
+      "rewards//mean": 0.74383544921875,
+      "rewards//std": 0.026944944635033607,
+      "step": 244
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.049,
+      "grad_norm": 2.460304021835327,
+      "kl": 0.08585498575121164,
+      "learning_rate": 9.962148412715463e-07,
+      "loss": 0.0034,
+      "num_tokens": 2118552.0,
+      "reward": 0.7547607421875,
+      "reward_std": 0.012553266249597073,
+      "rewards//mean": 0.7547607421875,
+      "rewards//std": 0.03701522946357727,
+      "step": 245
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0492,
+      "grad_norm": 2.2080492973327637,
+      "kl": 0.0854458324611187,
+      "learning_rate": 9.961757683914405e-07,
+      "loss": 0.0034,
+      "num_tokens": 2127248.0,
+      "reward": 0.69549560546875,
+      "reward_std": 0.01023833453655243,
+      "rewards//mean": 0.69549560546875,
+      "rewards//std": 0.04499489814043045,
+      "step": 246
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0494,
+      "grad_norm": 2.5779333114624023,
+      "kl": 0.07532225945033133,
+      "learning_rate": 9.961364956517946e-07,
+      "loss": 0.003,
+      "num_tokens": 2135896.0,
+      "reward": 0.75152587890625,
+      "reward_std": 0.012623130343854427,
+      "rewards//mean": 0.75152587890625,
+      "rewards//std": 0.047596026211977005,
+      "step": 247
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0496,
+      "grad_norm": 2.496809720993042,
+      "kl": 0.08910441934131086,
+      "learning_rate": 9.960970230684275e-07,
+      "loss": 0.0036,
+      "num_tokens": 2144536.0,
+      "reward": 0.72540283203125,
+      "reward_std": 0.013428730890154839,
+      "rewards//mean": 0.72540283203125,
+      "rewards//std": 0.04788045957684517,
+      "step": 248
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0498,
+      "grad_norm": 2.400987148284912,
+      "kl": 0.07812315365299582,
+      "learning_rate": 9.960573506572389e-07,
+      "loss": 0.0031,
+      "num_tokens": 2153104.0,
+      "reward": 0.75372314453125,
+      "reward_std": 0.011331569403409958,
+      "rewards//mean": 0.75372314453125,
+      "rewards//std": 0.02425442449748516,
+      "step": 249
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.05,
+      "grad_norm": 2.398341178894043,
+      "kl": 0.08263894519768655,
+      "learning_rate": 9.960174784342087e-07,
+      "loss": 0.0033,
+      "num_tokens": 2161736.0,
+      "reward": 0.7413330078125,
+      "reward_std": 0.008479075506329536,
+      "rewards//mean": 0.7413330078125,
+      "rewards//std": 0.03599345684051514,
+      "step": 250
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0502,
+      "grad_norm": 2.1341476440429688,
+      "kl": 0.08822295162826777,
+      "learning_rate": 9.959774064153975e-07,
+      "loss": 0.0035,
+      "num_tokens": 2170344.0,
+      "reward": 0.7177734375,
+      "reward_std": 0.008946191519498825,
+      "rewards//mean": 0.7177734375,
+      "rewards//std": 0.039610493928194046,
+      "step": 251
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0504,
+      "grad_norm": 2.428009271621704,
+      "kl": 0.08810280309990048,
+      "learning_rate": 9.959371346169465e-07,
+      "loss": 0.0035,
+      "num_tokens": 2179056.0,
+      "reward": 0.7904052734375,
+      "reward_std": 0.011364737525582314,
+      "rewards//mean": 0.7904052734375,
+      "rewards//std": 0.02987276203930378,
+      "step": 252
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0506,
+      "grad_norm": 2.2430484294891357,
+      "kl": 0.08622027351520956,
+      "learning_rate": 9.95896663055077e-07,
+      "loss": 0.0034,
+      "num_tokens": 2187640.0,
+      "reward": 0.71026611328125,
+      "reward_std": 0.009076209738850594,
+      "rewards//mean": 0.71026611328125,
+      "rewards//std": 0.04143089801073074,
+      "step": 253
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0508,
+      "grad_norm": 2.6776883602142334,
+      "kl": 0.08731716056354344,
+      "learning_rate": 9.958559917460907e-07,
+      "loss": 0.0035,
+      "num_tokens": 2196336.0,
+      "reward": 0.75714111328125,
+      "reward_std": 0.010459870100021362,
+      "rewards//mean": 0.75714111328125,
+      "rewards//std": 0.028164541348814964,
+      "step": 254
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.051,
+      "grad_norm": 2.2391140460968018,
+      "kl": 0.09201033553108573,
+      "learning_rate": 9.958151207063703e-07,
+      "loss": 0.0037,
+      "num_tokens": 2205024.0,
+      "reward": 0.7508544921875,
+      "reward_std": 0.010757776908576488,
+      "rewards//mean": 0.7508544921875,
+      "rewards//std": 0.036147892475128174,
+      "step": 255
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0512,
+      "grad_norm": 2.440457820892334,
+      "kl": 0.08760909363627434,
+      "learning_rate": 9.957740499523785e-07,
+      "loss": 0.0035,
+      "num_tokens": 2213608.0,
+      "reward": 0.75689697265625,
+      "reward_std": 0.013642119243741035,
+      "rewards//mean": 0.75689697265625,
+      "rewards//std": 0.04814309999346733,
+      "step": 256
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0514,
+      "grad_norm": 2.3290514945983887,
+      "kl": 0.09418594231829047,
+      "learning_rate": 9.957327795006588e-07,
+      "loss": 0.0038,
+      "num_tokens": 2222264.0,
+      "reward": 0.77484130859375,
+      "reward_std": 0.011928251013159752,
+      "rewards//mean": 0.77484130859375,
+      "rewards//std": 0.04047763720154762,
+      "step": 257
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0516,
+      "grad_norm": 2.230024576187134,
+      "kl": 0.08470788644626737,
+      "learning_rate": 9.956913093678348e-07,
+      "loss": 0.0034,
+      "num_tokens": 2230880.0,
+      "reward": 0.72161865234375,
+      "reward_std": 0.009003030136227608,
+      "rewards//mean": 0.72161865234375,
+      "rewards//std": 0.03698756545782089,
+      "step": 258
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0518,
+      "grad_norm": 2.324817419052124,
+      "kl": 0.11008088244125247,
+      "learning_rate": 9.956496395706105e-07,
+      "loss": 0.0044,
+      "num_tokens": 2239608.0,
+      "reward": 0.7716064453125,
+      "reward_std": 0.009707880206406116,
+      "rewards//mean": 0.7716064453125,
+      "rewards//std": 0.03827941045165062,
+      "step": 259
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.052,
+      "grad_norm": 2.1312897205352783,
+      "kl": 0.09451142069883645,
+      "learning_rate": 9.956077701257707e-07,
+      "loss": 0.0038,
+      "num_tokens": 2248296.0,
+      "reward": 0.7510986328125,
+      "reward_std": 0.008577214553952217,
+      "rewards//mean": 0.7510986328125,
+      "rewards//std": 0.026932932436466217,
+      "step": 260
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0522,
+      "grad_norm": 2.2232437133789062,
+      "kl": 0.11112384242005646,
+      "learning_rate": 9.955657010501806e-07,
+      "loss": 0.0044,
+      "num_tokens": 2256976.0,
+      "reward": 0.72271728515625,
+      "reward_std": 0.00968963373452425,
+      "rewards//mean": 0.72271728515625,
+      "rewards//std": 0.039515554904937744,
+      "step": 261
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0524,
+      "grad_norm": 2.0861966609954834,
+      "kl": 0.10497553274035454,
+      "learning_rate": 9.955234323607851e-07,
+      "loss": 0.0042,
+      "num_tokens": 2265672.0,
+      "reward": 0.76654052734375,
+      "reward_std": 0.009169444441795349,
+      "rewards//mean": 0.76654052734375,
+      "rewards//std": 0.035329725593328476,
+      "step": 262
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0526,
+      "grad_norm": 2.0662031173706055,
+      "kl": 0.09033403964713216,
+      "learning_rate": 9.954809640746105e-07,
+      "loss": 0.0036,
+      "num_tokens": 2274336.0,
+      "reward": 0.7734375,
+      "reward_std": 0.009483709000051022,
+      "rewards//mean": 0.7734375,
+      "rewards//std": 0.03672310337424278,
+      "step": 263
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0528,
+      "grad_norm": 2.315596103668213,
+      "kl": 0.09967878996394575,
+      "learning_rate": 9.954382962087627e-07,
+      "loss": 0.004,
+      "num_tokens": 2282984.0,
+      "reward": 0.78326416015625,
+      "reward_std": 0.011601641774177551,
+      "rewards//mean": 0.78326416015625,
+      "rewards//std": 0.032166559249162674,
+      "step": 264
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.053,
+      "grad_norm": 2.2670955657958984,
+      "kl": 0.10461910394951701,
+      "learning_rate": 9.953954287804284e-07,
+      "loss": 0.0042,
+      "num_tokens": 2291520.0,
+      "reward": 0.72039794921875,
+      "reward_std": 0.010187996551394463,
+      "rewards//mean": 0.72039794921875,
+      "rewards//std": 0.03597552329301834,
+      "step": 265
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0532,
+      "grad_norm": 2.1288392543792725,
+      "kl": 0.11212815390899777,
+      "learning_rate": 9.953523618068748e-07,
+      "loss": 0.0045,
+      "num_tokens": 2300080.0,
+      "reward": 0.73370361328125,
+      "reward_std": 0.00983397290110588,
+      "rewards//mean": 0.73370361328125,
+      "rewards//std": 0.02644364908337593,
+      "step": 266
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0534,
+      "grad_norm": 2.3853437900543213,
+      "kl": 0.12639968702569604,
+      "learning_rate": 9.95309095305449e-07,
+      "loss": 0.0051,
+      "num_tokens": 2308656.0,
+      "reward": 0.74774169921875,
+      "reward_std": 0.011285347864031792,
+      "rewards//mean": 0.74774169921875,
+      "rewards//std": 0.03584738075733185,
+      "step": 267
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0536,
+      "grad_norm": 2.105947494506836,
+      "kl": 0.10734104830771685,
+      "learning_rate": 9.952656292935788e-07,
+      "loss": 0.0043,
+      "num_tokens": 2317368.0,
+      "reward": 0.76641845703125,
+      "reward_std": 0.009294925257563591,
+      "rewards//mean": 0.76641845703125,
+      "rewards//std": 0.03633975610136986,
+      "step": 268
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0538,
+      "grad_norm": 2.4343624114990234,
+      "kl": 0.11141107883304358,
+      "learning_rate": 9.952219637887725e-07,
+      "loss": 0.0045,
+      "num_tokens": 2325992.0,
+      "reward": 0.74627685546875,
+      "reward_std": 0.013000589795410633,
+      "rewards//mean": 0.74627685546875,
+      "rewards//std": 0.0329802930355072,
+      "step": 269
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.054,
+      "grad_norm": 2.114595890045166,
+      "kl": 0.11248049885034561,
+      "learning_rate": 9.951780988086183e-07,
+      "loss": 0.0045,
+      "num_tokens": 2334616.0,
+      "reward": 0.7659912109375,
+      "reward_std": 0.008284421637654305,
+      "rewards//mean": 0.7659912109375,
+      "rewards//std": 0.024673106148838997,
+      "step": 270
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0542,
+      "grad_norm": 2.3597919940948486,
+      "kl": 0.10822209948673844,
+      "learning_rate": 9.95134034370785e-07,
+      "loss": 0.0043,
+      "num_tokens": 2343264.0,
+      "reward": 0.75213623046875,
+      "reward_std": 0.008110977709293365,
+      "rewards//mean": 0.75213623046875,
+      "rewards//std": 0.02558121271431446,
+      "step": 271
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0544,
+      "grad_norm": 2.342583417892456,
+      "kl": 0.10092776664532721,
+      "learning_rate": 9.95089770493022e-07,
+      "loss": 0.004,
+      "num_tokens": 2351936.0,
+      "reward": 0.73480224609375,
+      "reward_std": 0.009873945266008377,
+      "rewards//mean": 0.73480224609375,
+      "rewards//std": 0.04320269823074341,
+      "step": 272
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0546,
+      "grad_norm": 2.247030019760132,
+      "kl": 0.11563214473426342,
+      "learning_rate": 9.950453071931588e-07,
+      "loss": 0.0046,
+      "num_tokens": 2360560.0,
+      "reward": 0.7540283203125,
+      "reward_std": 0.009906873106956482,
+      "rewards//mean": 0.7540283203125,
+      "rewards//std": 0.03898783028125763,
+      "step": 273
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0548,
+      "grad_norm": 2.341111660003662,
+      "kl": 0.11460674507543445,
+      "learning_rate": 9.950006444891048e-07,
+      "loss": 0.0046,
+      "num_tokens": 2369160.0,
+      "reward": 0.71240234375,
+      "reward_std": 0.010771198198199272,
+      "rewards//mean": 0.71240234375,
+      "rewards//std": 0.033519841730594635,
+      "step": 274
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.055,
+      "grad_norm": 2.476717948913574,
+      "kl": 0.10514741018414497,
+      "learning_rate": 9.949557823988506e-07,
+      "loss": 0.0042,
+      "num_tokens": 2377840.0,
+      "reward": 0.7381591796875,
+      "reward_std": 0.01162101048976183,
+      "rewards//mean": 0.7381591796875,
+      "rewards//std": 0.03822242096066475,
+      "step": 275
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0552,
+      "grad_norm": 2.486793041229248,
+      "kl": 0.11561599606648088,
+      "learning_rate": 9.949107209404663e-07,
+      "loss": 0.0046,
+      "num_tokens": 2386504.0,
+      "reward": 0.7628173828125,
+      "reward_std": 0.008916687220335007,
+      "rewards//mean": 0.7628173828125,
+      "rewards//std": 0.021748732775449753,
+      "step": 276
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0554,
+      "grad_norm": 2.1925792694091797,
+      "kl": 0.11504006339237094,
+      "learning_rate": 9.94865460132103e-07,
+      "loss": 0.0046,
+      "num_tokens": 2395128.0,
+      "reward": 0.75469970703125,
+      "reward_std": 0.013766275718808174,
+      "rewards//mean": 0.75469970703125,
+      "rewards//std": 0.03127463534474373,
+      "step": 277
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0556,
+      "grad_norm": 2.2684152126312256,
+      "kl": 0.1263164347037673,
+      "learning_rate": 9.948199999919912e-07,
+      "loss": 0.0051,
+      "num_tokens": 2403824.0,
+      "reward": 0.71728515625,
+      "reward_std": 0.008052065037190914,
+      "rewards//mean": 0.71728515625,
+      "rewards//std": 0.031269371509552,
+      "step": 278
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0558,
+      "grad_norm": 2.307429552078247,
+      "kl": 0.1207714956253767,
+      "learning_rate": 9.947743405384428e-07,
+      "loss": 0.0048,
+      "num_tokens": 2412440.0,
+      "reward": 0.74017333984375,
+      "reward_std": 0.010619981214404106,
+      "rewards//mean": 0.74017333984375,
+      "rewards//std": 0.037441518157720566,
+      "step": 279
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.056,
+      "grad_norm": 2.2332217693328857,
+      "kl": 0.11981469811871648,
+      "learning_rate": 9.947284817898492e-07,
+      "loss": 0.0048,
+      "num_tokens": 2421072.0,
+      "reward": 0.7069091796875,
+      "reward_std": 0.01122340653091669,
+      "rewards//mean": 0.7069091796875,
+      "rewards//std": 0.03928488865494728,
+      "step": 280
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0562,
+      "grad_norm": 2.250086545944214,
+      "kl": 0.12764271907508373,
+      "learning_rate": 9.946824237646824e-07,
+      "loss": 0.0051,
+      "num_tokens": 2429736.0,
+      "reward": 0.744873046875,
+      "reward_std": 0.010046460665762424,
+      "rewards//mean": 0.744873046875,
+      "rewards//std": 0.03565817326307297,
+      "step": 281
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0564,
+      "grad_norm": 2.4999492168426514,
+      "kl": 0.12754053669050336,
+      "learning_rate": 9.946361664814943e-07,
+      "loss": 0.0051,
+      "num_tokens": 2438336.0,
+      "reward": 0.73980712890625,
+      "reward_std": 0.01213806588202715,
+      "rewards//mean": 0.73980712890625,
+      "rewards//std": 0.021334391087293625,
+      "step": 282
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0566,
+      "grad_norm": 2.4783177375793457,
+      "kl": 0.1202986822463572,
+      "learning_rate": 9.945897099589173e-07,
+      "loss": 0.0048,
+      "num_tokens": 2446944.0,
+      "reward": 0.72564697265625,
+      "reward_std": 0.011273622512817383,
+      "rewards//mean": 0.72564697265625,
+      "rewards//std": 0.027434322983026505,
+      "step": 283
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0568,
+      "grad_norm": 2.176853895187378,
+      "kl": 0.12868809327483177,
+      "learning_rate": 9.945430542156646e-07,
+      "loss": 0.0051,
+      "num_tokens": 2455528.0,
+      "reward": 0.76263427734375,
+      "reward_std": 0.01164940744638443,
+      "rewards//mean": 0.76263427734375,
+      "rewards//std": 0.02524586021900177,
+      "step": 284
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.057,
+      "grad_norm": 2.0553863048553467,
+      "kl": 0.11887755850329995,
+      "learning_rate": 9.944961992705286e-07,
+      "loss": 0.0048,
+      "num_tokens": 2464104.0,
+      "reward": 0.7296142578125,
+      "reward_std": 0.007252207491546869,
+      "rewards//mean": 0.7296142578125,
+      "rewards//std": 0.03891632333397865,
+      "step": 285
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0572,
+      "grad_norm": 2.3183748722076416,
+      "kl": 0.11394174303859472,
+      "learning_rate": 9.944491451423827e-07,
+      "loss": 0.0046,
+      "num_tokens": 2472768.0,
+      "reward": 0.77862548828125,
+      "reward_std": 0.010248874314129353,
+      "rewards//mean": 0.77862548828125,
+      "rewards//std": 0.021373379975557327,
+      "step": 286
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0574,
+      "grad_norm": 2.401534080505371,
+      "kl": 0.12894047936424613,
+      "learning_rate": 9.944018918501805e-07,
+      "loss": 0.0052,
+      "num_tokens": 2481432.0,
+      "reward": 0.73358154296875,
+      "reward_std": 0.009259985759854317,
+      "rewards//mean": 0.73358154296875,
+      "rewards//std": 0.034647442400455475,
+      "step": 287
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0576,
+      "grad_norm": 2.3681552410125732,
+      "kl": 0.13058429351076484,
+      "learning_rate": 9.94354439412955e-07,
+      "loss": 0.0052,
+      "num_tokens": 2490120.0,
+      "reward": 0.73504638671875,
+      "reward_std": 0.010098876431584358,
+      "rewards//mean": 0.73504638671875,
+      "rewards//std": 0.023883353918790817,
+      "step": 288
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0578,
+      "grad_norm": 2.335580348968506,
+      "kl": 0.13276920886710286,
+      "learning_rate": 9.943067878498209e-07,
+      "loss": 0.0053,
+      "num_tokens": 2498832.0,
+      "reward": 0.7470703125,
+      "reward_std": 0.009666764177381992,
+      "rewards//mean": 0.7470703125,
+      "rewards//std": 0.034583210945129395,
+      "step": 289
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.058,
+      "grad_norm": 3.374286651611328,
+      "kl": 0.1374687599018216,
+      "learning_rate": 9.942589371799714e-07,
+      "loss": 0.0055,
+      "num_tokens": 2507544.0,
+      "reward": 0.75628662109375,
+      "reward_std": 0.007982924580574036,
+      "rewards//mean": 0.75628662109375,
+      "rewards//std": 0.03333229571580887,
+      "step": 290
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0582,
+      "grad_norm": 2.4196667671203613,
+      "kl": 0.13316209614276886,
+      "learning_rate": 9.94210887422681e-07,
+      "loss": 0.0053,
+      "num_tokens": 2516200.0,
+      "reward": 0.75543212890625,
+      "reward_std": 0.010620934888720512,
+      "rewards//mean": 0.75543212890625,
+      "rewards//std": 0.04492587968707085,
+      "step": 291
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0584,
+      "grad_norm": 2.6077113151550293,
+      "kl": 0.13197654951363802,
+      "learning_rate": 9.941626385973047e-07,
+      "loss": 0.0053,
+      "num_tokens": 2524768.0,
+      "reward": 0.75946044921875,
+      "reward_std": 0.00954778678715229,
+      "rewards//mean": 0.75946044921875,
+      "rewards//std": 0.028338147327303886,
+      "step": 292
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0586,
+      "grad_norm": 2.331041097640991,
+      "kl": 0.11738697905093431,
+      "learning_rate": 9.941141907232763e-07,
+      "loss": 0.0047,
+      "num_tokens": 2533440.0,
+      "reward": 0.753662109375,
+      "reward_std": 0.009502648375928402,
+      "rewards//mean": 0.753662109375,
+      "rewards//std": 0.03564458340406418,
+      "step": 293
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0588,
+      "grad_norm": 2.3681371212005615,
+      "kl": 0.1467471462674439,
+      "learning_rate": 9.94065543820111e-07,
+      "loss": 0.0059,
+      "num_tokens": 2542128.0,
+      "reward": 0.79052734375,
+      "reward_std": 0.010154277086257935,
+      "rewards//mean": 0.79052734375,
+      "rewards//std": 0.02691522240638733,
+      "step": 294
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.059,
+      "grad_norm": 2.214519739151001,
+      "kl": 0.14778105821460485,
+      "learning_rate": 9.94016697907404e-07,
+      "loss": 0.0059,
+      "num_tokens": 2550680.0,
+      "reward": 0.73980712890625,
+      "reward_std": 0.012334965169429779,
+      "rewards//mean": 0.73980712890625,
+      "rewards//std": 0.031967345625162125,
+      "step": 295
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0592,
+      "grad_norm": 2.359705686569214,
+      "kl": 0.135012770537287,
+      "learning_rate": 9.9396765300483e-07,
+      "loss": 0.0054,
+      "num_tokens": 2559424.0,
+      "reward": 0.7669677734375,
+      "reward_std": 0.008275476284325123,
+      "rewards//mean": 0.7669677734375,
+      "rewards//std": 0.03668495640158653,
+      "step": 296
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0594,
+      "grad_norm": 2.2671761512756348,
+      "kl": 0.12718097073957324,
+      "learning_rate": 9.939184091321444e-07,
+      "loss": 0.0051,
+      "num_tokens": 2568040.0,
+      "reward": 0.78363037109375,
+      "reward_std": 0.007923097349703312,
+      "rewards//mean": 0.78363037109375,
+      "rewards//std": 0.026158751919865608,
+      "step": 297
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0596,
+      "grad_norm": 2.3568453788757324,
+      "kl": 0.14283269178122282,
+      "learning_rate": 9.938689663091827e-07,
+      "loss": 0.0057,
+      "num_tokens": 2576776.0,
+      "reward": 0.7615966796875,
+      "reward_std": 0.00952947698533535,
+      "rewards//mean": 0.7615966796875,
+      "rewards//std": 0.034827250987291336,
+      "step": 298
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0598,
+      "grad_norm": 2.3118374347686768,
+      "kl": 0.13765425560995936,
+      "learning_rate": 9.938193245558604e-07,
+      "loss": 0.0055,
+      "num_tokens": 2585392.0,
+      "reward": 0.72393798828125,
+      "reward_std": 0.011698382906615734,
+      "rewards//mean": 0.72393798828125,
+      "rewards//std": 0.0432247668504715,
+      "step": 299
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.06,
+      "grad_norm": 2.3777027130126953,
+      "kl": 0.13236781675368547,
+      "learning_rate": 9.937694838921733e-07,
+      "loss": 0.0053,
+      "num_tokens": 2594032.0,
+      "reward": 0.7261962890625,
+      "reward_std": 0.007660794071853161,
+      "rewards//mean": 0.7261962890625,
+      "rewards//std": 0.02233387529850006,
+      "step": 300
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0602,
+      "grad_norm": 2.3749730587005615,
+      "kl": 0.14261657977476716,
+      "learning_rate": 9.93719444338197e-07,
+      "loss": 0.0057,
+      "num_tokens": 2602736.0,
+      "reward": 0.725830078125,
+      "reward_std": 0.012275228276848793,
+      "rewards//mean": 0.725830078125,
+      "rewards//std": 0.044357601553201675,
+      "step": 301
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0604,
+      "grad_norm": 2.3030948638916016,
+      "kl": 0.14758447650820017,
+      "learning_rate": 9.936692059140878e-07,
+      "loss": 0.0059,
+      "num_tokens": 2611384.0,
+      "reward": 0.7725830078125,
+      "reward_std": 0.008051641285419464,
+      "rewards//mean": 0.7725830078125,
+      "rewards//std": 0.02942551113665104,
+      "step": 302
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0606,
+      "grad_norm": 2.4487171173095703,
+      "kl": 0.13396706525236368,
+      "learning_rate": 9.936187686400814e-07,
+      "loss": 0.0054,
+      "num_tokens": 2620152.0,
+      "reward": 0.75701904296875,
+      "reward_std": 0.009920504875481129,
+      "rewards//mean": 0.75701904296875,
+      "rewards//std": 0.03601841628551483,
+      "step": 303
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0608,
+      "grad_norm": 2.4276504516601562,
+      "kl": 0.14364641439169645,
+      "learning_rate": 9.93568132536494e-07,
+      "loss": 0.0057,
+      "num_tokens": 2628816.0,
+      "reward": 0.75531005859375,
+      "reward_std": 0.01256647054105997,
+      "rewards//mean": 0.75531005859375,
+      "rewards//std": 0.030956953763961792,
+      "step": 304
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.061,
+      "grad_norm": 2.934332847595215,
+      "kl": 0.1464973008260131,
+      "learning_rate": 9.935172976237217e-07,
+      "loss": 0.0059,
+      "num_tokens": 2637496.0,
+      "reward": 0.75006103515625,
+      "reward_std": 0.010656114667654037,
+      "rewards//mean": 0.75006103515625,
+      "rewards//std": 0.016754386946558952,
+      "step": 305
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0612,
+      "grad_norm": 2.670234203338623,
+      "kl": 0.13851862819865346,
+      "learning_rate": 9.93466263922241e-07,
+      "loss": 0.0055,
+      "num_tokens": 2646104.0,
+      "reward": 0.7593994140625,
+      "reward_std": 0.010100295767188072,
+      "rewards//mean": 0.7593994140625,
+      "rewards//std": 0.028921151533722878,
+      "step": 306
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0614,
+      "grad_norm": 2.3960487842559814,
+      "kl": 0.14857173897325993,
+      "learning_rate": 9.934150314526083e-07,
+      "loss": 0.0059,
+      "num_tokens": 2654744.0,
+      "reward": 0.769775390625,
+      "reward_std": 0.00879153236746788,
+      "rewards//mean": 0.769775390625,
+      "rewards//std": 0.015975650399923325,
+      "step": 307
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0616,
+      "grad_norm": 2.3883743286132812,
+      "kl": 0.14142659353092313,
+      "learning_rate": 9.933636002354599e-07,
+      "loss": 0.0057,
+      "num_tokens": 2663376.0,
+      "reward": 0.71966552734375,
+      "reward_std": 0.009030375629663467,
+      "rewards//mean": 0.71966552734375,
+      "rewards//std": 0.02294394187629223,
+      "step": 308
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0618,
+      "grad_norm": 2.4159791469573975,
+      "kl": 0.13685584999620914,
+      "learning_rate": 9.933119702915124e-07,
+      "loss": 0.0055,
+      "num_tokens": 2671952.0,
+      "reward": 0.7314453125,
+      "reward_std": 0.009831303730607033,
+      "rewards//mean": 0.7314453125,
+      "rewards//std": 0.025836244225502014,
+      "step": 309
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.062,
+      "grad_norm": 2.4496707916259766,
+      "kl": 0.14309891825541854,
+      "learning_rate": 9.93260141641562e-07,
+      "loss": 0.0057,
+      "num_tokens": 2680624.0,
+      "reward": 0.74346923828125,
+      "reward_std": 0.010039031505584717,
+      "rewards//mean": 0.74346923828125,
+      "rewards//std": 0.029430333524942398,
+      "step": 310
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0622,
+      "grad_norm": 2.4669125080108643,
+      "kl": 0.1311500621959567,
+      "learning_rate": 9.932081143064858e-07,
+      "loss": 0.0052,
+      "num_tokens": 2689176.0,
+      "reward": 0.7725830078125,
+      "reward_std": 0.010175235569477081,
+      "rewards//mean": 0.7725830078125,
+      "rewards//std": 0.03233535587787628,
+      "step": 311
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0624,
+      "grad_norm": 2.6372668743133545,
+      "kl": 0.14883148716762662,
+      "learning_rate": 9.931558883072402e-07,
+      "loss": 0.006,
+      "num_tokens": 2697864.0,
+      "reward": 0.7506103515625,
+      "reward_std": 0.009078023955225945,
+      "rewards//mean": 0.7506103515625,
+      "rewards//std": 0.02745617926120758,
+      "step": 312
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0626,
+      "grad_norm": 2.422214984893799,
+      "kl": 0.13642366975545883,
+      "learning_rate": 9.931034636648616e-07,
+      "loss": 0.0055,
+      "num_tokens": 2706480.0,
+      "reward": 0.71978759765625,
+      "reward_std": 0.008253131061792374,
+      "rewards//mean": 0.71978759765625,
+      "rewards//std": 0.02018669806420803,
+      "step": 313
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0628,
+      "grad_norm": 2.507030487060547,
+      "kl": 0.1547061800956726,
+      "learning_rate": 9.930508404004666e-07,
+      "loss": 0.0062,
+      "num_tokens": 2715056.0,
+      "reward": 0.7611083984375,
+      "reward_std": 0.008002392947673798,
+      "rewards//mean": 0.7611083984375,
+      "rewards//std": 0.03237839788198471,
+      "step": 314
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.063,
+      "grad_norm": 2.4126579761505127,
+      "kl": 0.14070541644468904,
+      "learning_rate": 9.929980185352525e-07,
+      "loss": 0.0056,
+      "num_tokens": 2723632.0,
+      "reward": 0.75,
+      "reward_std": 0.007926436141133308,
+      "rewards//mean": 0.75,
+      "rewards//std": 0.03435836732387543,
+      "step": 315
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0632,
+      "grad_norm": 2.4670326709747314,
+      "kl": 0.13856445252895355,
+      "learning_rate": 9.929449980904951e-07,
+      "loss": 0.0055,
+      "num_tokens": 2732264.0,
+      "reward": 0.728759765625,
+      "reward_std": 0.00800327304750681,
+      "rewards//mean": 0.728759765625,
+      "rewards//std": 0.032090287655591965,
+      "step": 316
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0634,
+      "grad_norm": 2.4115591049194336,
+      "kl": 0.1344844363629818,
+      "learning_rate": 9.928917790875516e-07,
+      "loss": 0.0054,
+      "num_tokens": 2740960.0,
+      "reward": 0.7650146484375,
+      "reward_std": 0.008280552923679352,
+      "rewards//mean": 0.7650146484375,
+      "rewards//std": 0.030203351750969887,
+      "step": 317
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0636,
+      "grad_norm": 2.355797290802002,
+      "kl": 0.14118980802595615,
+      "learning_rate": 9.928383615478586e-07,
+      "loss": 0.0056,
+      "num_tokens": 2749528.0,
+      "reward": 0.75140380859375,
+      "reward_std": 0.010042013600468636,
+      "rewards//mean": 0.75140380859375,
+      "rewards//std": 0.023482587188482285,
+      "step": 318
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0638,
+      "grad_norm": 2.4152274131774902,
+      "kl": 0.14794302312657237,
+      "learning_rate": 9.927847454929322e-07,
+      "loss": 0.0059,
+      "num_tokens": 2758176.0,
+      "reward": 0.7567138671875,
+      "reward_std": 0.007422657683491707,
+      "rewards//mean": 0.7567138671875,
+      "rewards//std": 0.024572281166911125,
+      "step": 319
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.064,
+      "grad_norm": 2.4957797527313232,
+      "kl": 0.14901667507365346,
+      "learning_rate": 9.927309309443695e-07,
+      "loss": 0.006,
+      "num_tokens": 2766760.0,
+      "reward": 0.76763916015625,
+      "reward_std": 0.008163049817085266,
+      "rewards//mean": 0.76763916015625,
+      "rewards//std": 0.03143540024757385,
+      "step": 320
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0642,
+      "grad_norm": 2.529953956604004,
+      "kl": 0.14068567799404263,
+      "learning_rate": 9.926769179238464e-07,
+      "loss": 0.0056,
+      "num_tokens": 2775344.0,
+      "reward": 0.75244140625,
+      "reward_std": 0.008808191865682602,
+      "rewards//mean": 0.75244140625,
+      "rewards//std": 0.03251486271619797,
+      "step": 321
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0644,
+      "grad_norm": 2.4482409954071045,
+      "kl": 0.13553015561774373,
+      "learning_rate": 9.926227064531199e-07,
+      "loss": 0.0054,
+      "num_tokens": 2783880.0,
+      "reward": 0.75732421875,
+      "reward_std": 0.009315178729593754,
+      "rewards//mean": 0.75732421875,
+      "rewards//std": 0.023255953565239906,
+      "step": 322
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0646,
+      "grad_norm": 2.410548210144043,
+      "kl": 0.14313685148954391,
+      "learning_rate": 9.925682965540263e-07,
+      "loss": 0.0057,
+      "num_tokens": 2792480.0,
+      "reward": 0.7581787109375,
+      "reward_std": 0.009847695007920265,
+      "rewards//mean": 0.7581787109375,
+      "rewards//std": 0.030062692239880562,
+      "step": 323
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0648,
+      "grad_norm": 2.4038262367248535,
+      "kl": 0.1418413147330284,
+      "learning_rate": 9.925136882484815e-07,
+      "loss": 0.0057,
+      "num_tokens": 2801088.0,
+      "reward": 0.7283935546875,
+      "reward_std": 0.0071106343530118465,
+      "rewards//mean": 0.7283935546875,
+      "rewards//std": 0.029023557901382446,
+      "step": 324
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.065,
+      "grad_norm": 2.455026865005493,
+      "kl": 0.14870468340814114,
+      "learning_rate": 9.92458881558482e-07,
+      "loss": 0.0059,
+      "num_tokens": 2809768.0,
+      "reward": 0.752197265625,
+      "reward_std": 0.01029081642627716,
+      "rewards//mean": 0.752197265625,
+      "rewards//std": 0.027532432228326797,
+      "step": 325
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0652,
+      "grad_norm": 2.627734661102295,
+      "kl": 0.14815522264689207,
+      "learning_rate": 9.92403876506104e-07,
+      "loss": 0.0059,
+      "num_tokens": 2818328.0,
+      "reward": 0.76214599609375,
+      "reward_std": 0.007829036563634872,
+      "rewards//mean": 0.76214599609375,
+      "rewards//std": 0.03401203081011772,
+      "step": 326
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0654,
+      "grad_norm": 2.2990593910217285,
+      "kl": 0.1547041591256857,
+      "learning_rate": 9.923486731135033e-07,
+      "loss": 0.0062,
+      "num_tokens": 2826984.0,
+      "reward": 0.73211669921875,
+      "reward_std": 0.009885728359222412,
+      "rewards//mean": 0.73211669921875,
+      "rewards//std": 0.04717273265123367,
+      "step": 327
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0656,
+      "grad_norm": 2.4028618335723877,
+      "kl": 0.14704480860382318,
+      "learning_rate": 9.922932714029163e-07,
+      "loss": 0.0059,
+      "num_tokens": 2835544.0,
+      "reward": 0.77032470703125,
+      "reward_std": 0.008336121216416359,
+      "rewards//mean": 0.77032470703125,
+      "rewards//std": 0.022044189274311066,
+      "step": 328
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0658,
+      "grad_norm": 2.481088638305664,
+      "kl": 0.15337435388937593,
+      "learning_rate": 9.92237671396658e-07,
+      "loss": 0.0061,
+      "num_tokens": 2844184.0,
+      "reward": 0.78033447265625,
+      "reward_std": 0.009491624310612679,
+      "rewards//mean": 0.78033447265625,
+      "rewards//std": 0.025861937552690506,
+      "step": 329
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.066,
+      "grad_norm": 2.3463478088378906,
+      "kl": 0.15274662571027875,
+      "learning_rate": 9.921818731171248e-07,
+      "loss": 0.0061,
+      "num_tokens": 2852824.0,
+      "reward": 0.7298583984375,
+      "reward_std": 0.007879456505179405,
+      "rewards//mean": 0.7298583984375,
+      "rewards//std": 0.01685134693980217,
+      "step": 330
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0662,
+      "grad_norm": 2.4336092472076416,
+      "kl": 0.16218566242605448,
+      "learning_rate": 9.921258765867919e-07,
+      "loss": 0.0065,
+      "num_tokens": 2861568.0,
+      "reward": 0.762451171875,
+      "reward_std": 0.008910413831472397,
+      "rewards//mean": 0.762451171875,
+      "rewards//std": 0.027558811008930206,
+      "step": 331
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0664,
+      "grad_norm": 2.687967538833618,
+      "kl": 0.18005005735903978,
+      "learning_rate": 9.920696818282147e-07,
+      "loss": 0.0072,
+      "num_tokens": 2870168.0,
+      "reward": 0.7740478515625,
+      "reward_std": 0.006842412054538727,
+      "rewards//mean": 0.7740478515625,
+      "rewards//std": 0.025163955986499786,
+      "step": 332
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0666,
+      "grad_norm": 2.426886796951294,
+      "kl": 0.1458109705708921,
+      "learning_rate": 9.920132888640284e-07,
+      "loss": 0.0058,
+      "num_tokens": 2878752.0,
+      "reward": 0.73150634765625,
+      "reward_std": 0.008681036531925201,
+      "rewards//mean": 0.73150634765625,
+      "rewards//std": 0.03020353987812996,
+      "step": 333
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0668,
+      "grad_norm": 2.489117383956909,
+      "kl": 0.16056799283251166,
+      "learning_rate": 9.919566977169485e-07,
+      "loss": 0.0064,
+      "num_tokens": 2887504.0,
+      "reward": 0.73114013671875,
+      "reward_std": 0.010405587032437325,
+      "rewards//mean": 0.73114013671875,
+      "rewards//std": 0.034376371651887894,
+      "step": 334
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.067,
+      "grad_norm": 2.3607702255249023,
+      "kl": 0.14290315005928278,
+      "learning_rate": 9.918999084097694e-07,
+      "loss": 0.0057,
+      "num_tokens": 2896176.0,
+      "reward": 0.718017578125,
+      "reward_std": 0.007494831457734108,
+      "rewards//mean": 0.718017578125,
+      "rewards//std": 0.022605039179325104,
+      "step": 335
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0672,
+      "grad_norm": 2.3309366703033447,
+      "kl": 0.13763791276142,
+      "learning_rate": 9.91842920965366e-07,
+      "loss": 0.0055,
+      "num_tokens": 2904808.0,
+      "reward": 0.7408447265625,
+      "reward_std": 0.008373770862817764,
+      "rewards//mean": 0.7408447265625,
+      "rewards//std": 0.028170250356197357,
+      "step": 336
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0674,
+      "grad_norm": 2.3058981895446777,
+      "kl": 0.15147208329290152,
+      "learning_rate": 9.91785735406693e-07,
+      "loss": 0.0061,
+      "num_tokens": 2913544.0,
+      "reward": 0.7340087890625,
+      "reward_std": 0.007901324890553951,
+      "rewards//mean": 0.7340087890625,
+      "rewards//std": 0.033354826271533966,
+      "step": 337
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0676,
+      "grad_norm": 2.5370399951934814,
+      "kl": 0.16497230855748057,
+      "learning_rate": 9.917283517567843e-07,
+      "loss": 0.0066,
+      "num_tokens": 2922208.0,
+      "reward": 0.74591064453125,
+      "reward_std": 0.007384471595287323,
+      "rewards//mean": 0.74591064453125,
+      "rewards//std": 0.025481605902314186,
+      "step": 338
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0678,
+      "grad_norm": 2.6054527759552,
+      "kl": 0.17215240187942982,
+      "learning_rate": 9.916707700387545e-07,
+      "loss": 0.0069,
+      "num_tokens": 2930904.0,
+      "reward": 0.7353515625,
+      "reward_std": 0.008231500163674355,
+      "rewards//mean": 0.7353515625,
+      "rewards//std": 0.03357398882508278,
+      "step": 339
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.068,
+      "grad_norm": 2.465613603591919,
+      "kl": 0.1483508301898837,
+      "learning_rate": 9.916129902757974e-07,
+      "loss": 0.0059,
+      "num_tokens": 2939600.0,
+      "reward": 0.7542724609375,
+      "reward_std": 0.00948614813387394,
+      "rewards//mean": 0.7542724609375,
+      "rewards//std": 0.022922571748495102,
+      "step": 340
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0682,
+      "grad_norm": 2.374268054962158,
+      "kl": 0.17610491486266255,
+      "learning_rate": 9.915550124911866e-07,
+      "loss": 0.007,
+      "num_tokens": 2948432.0,
+      "reward": 0.74737548828125,
+      "reward_std": 0.0069858599454164505,
+      "rewards//mean": 0.74737548828125,
+      "rewards//std": 0.022756462916731834,
+      "step": 341
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0684,
+      "grad_norm": 2.5809059143066406,
+      "kl": 0.15459762560203671,
+      "learning_rate": 9.914968367082755e-07,
+      "loss": 0.0062,
+      "num_tokens": 2957032.0,
+      "reward": 0.782470703125,
+      "reward_std": 0.008777000941336155,
+      "rewards//mean": 0.782470703125,
+      "rewards//std": 0.016156556084752083,
+      "step": 342
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0686,
+      "grad_norm": 2.8978984355926514,
+      "kl": 0.15157524775713682,
+      "learning_rate": 9.914384629504973e-07,
+      "loss": 0.0061,
+      "num_tokens": 2965680.0,
+      "reward": 0.75439453125,
+      "reward_std": 0.009697170928120613,
+      "rewards//mean": 0.75439453125,
+      "rewards//std": 0.0314624160528183,
+      "step": 343
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0688,
+      "grad_norm": 2.5209693908691406,
+      "kl": 0.16175976116210222,
+      "learning_rate": 9.913798912413652e-07,
+      "loss": 0.0065,
+      "num_tokens": 2974304.0,
+      "reward": 0.70819091796875,
+      "reward_std": 0.008540173061192036,
+      "rewards//mean": 0.70819091796875,
+      "rewards//std": 0.03589380159974098,
+      "step": 344
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.069,
+      "grad_norm": 2.5906119346618652,
+      "kl": 0.15957222785800695,
+      "learning_rate": 9.913211216044713e-07,
+      "loss": 0.0064,
+      "num_tokens": 2982920.0,
+      "reward": 0.72332763671875,
+      "reward_std": 0.008108312264084816,
+      "rewards//mean": 0.72332763671875,
+      "rewards//std": 0.038860805332660675,
+      "step": 345
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0692,
+      "grad_norm": 2.638352870941162,
+      "kl": 0.14657947700470686,
+      "learning_rate": 9.912621540634886e-07,
+      "loss": 0.0059,
+      "num_tokens": 2991640.0,
+      "reward": 0.76904296875,
+      "reward_std": 0.006945888977497816,
+      "rewards//mean": 0.76904296875,
+      "rewards//std": 0.024503495544195175,
+      "step": 346
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0694,
+      "grad_norm": 2.2758610248565674,
+      "kl": 0.16795516945421696,
+      "learning_rate": 9.91202988642169e-07,
+      "loss": 0.0067,
+      "num_tokens": 3000224.0,
+      "reward": 0.7584228515625,
+      "reward_std": 0.006969330366700888,
+      "rewards//mean": 0.7584228515625,
+      "rewards//std": 0.02684059739112854,
+      "step": 347
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0696,
+      "grad_norm": 2.6884002685546875,
+      "kl": 0.16269545536488295,
+      "learning_rate": 9.911436253643443e-07,
+      "loss": 0.0065,
+      "num_tokens": 3008880.0,
+      "reward": 0.74200439453125,
+      "reward_std": 0.010088170878589153,
+      "rewards//mean": 0.74200439453125,
+      "rewards//std": 0.02847137860953808,
+      "step": 348
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0698,
+      "grad_norm": 2.5450334548950195,
+      "kl": 0.14913112577050924,
+      "learning_rate": 9.91084064253926e-07,
+      "loss": 0.006,
+      "num_tokens": 3017536.0,
+      "reward": 0.7344970703125,
+      "reward_std": 0.0112111521884799,
+      "rewards//mean": 0.7344970703125,
+      "rewards//std": 0.029528219252824783,
+      "step": 349
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.07,
+      "grad_norm": 2.740511894226074,
+      "kl": 0.16483196150511503,
+      "learning_rate": 9.910243053349055e-07,
+      "loss": 0.0066,
+      "num_tokens": 3026160.0,
+      "reward": 0.7740478515625,
+      "reward_std": 0.010326296091079712,
+      "rewards//mean": 0.7740478515625,
+      "rewards//std": 0.02729693055152893,
+      "step": 350
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0702,
+      "grad_norm": 2.612835168838501,
+      "kl": 0.1619471926242113,
+      "learning_rate": 9.909643486313533e-07,
+      "loss": 0.0065,
+      "num_tokens": 3034792.0,
+      "reward": 0.7838134765625,
+      "reward_std": 0.008278329856693745,
+      "rewards//mean": 0.7838134765625,
+      "rewards//std": 0.019749755039811134,
+      "step": 351
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0704,
+      "grad_norm": 4.1898603439331055,
+      "kl": 0.16626066528260708,
+      "learning_rate": 9.909041941674204e-07,
+      "loss": 0.0067,
+      "num_tokens": 3043432.0,
+      "reward": 0.759033203125,
+      "reward_std": 0.007817190140485764,
+      "rewards//mean": 0.759033203125,
+      "rewards//std": 0.026068393141031265,
+      "step": 352
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0706,
+      "grad_norm": 2.5477583408355713,
+      "kl": 0.164810037240386,
+      "learning_rate": 9.908438419673366e-07,
+      "loss": 0.0066,
+      "num_tokens": 3052008.0,
+      "reward": 0.7667236328125,
+      "reward_std": 0.011191527359187603,
+      "rewards//mean": 0.7667236328125,
+      "rewards//std": 0.022867033258080482,
+      "step": 353
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0708,
+      "grad_norm": 2.383877992630005,
+      "kl": 0.1532201822847128,
+      "learning_rate": 9.90783292055412e-07,
+      "loss": 0.0061,
+      "num_tokens": 3060680.0,
+      "reward": 0.76751708984375,
+      "reward_std": 0.008705323562026024,
+      "rewards//mean": 0.76751708984375,
+      "rewards//std": 0.02952020801603794,
+      "step": 354
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.071,
+      "grad_norm": 2.4674792289733887,
+      "kl": 0.16015919670462608,
+      "learning_rate": 9.907225444560361e-07,
+      "loss": 0.0064,
+      "num_tokens": 3069312.0,
+      "reward": 0.77276611328125,
+      "reward_std": 0.008959580212831497,
+      "rewards//mean": 0.77276611328125,
+      "rewards//std": 0.026305902749300003,
+      "step": 355
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0712,
+      "grad_norm": 2.8221089839935303,
+      "kl": 0.16012147534638643,
+      "learning_rate": 9.90661599193678e-07,
+      "loss": 0.0064,
+      "num_tokens": 3078032.0,
+      "reward": 0.74908447265625,
+      "reward_std": 0.009289544075727463,
+      "rewards//mean": 0.74908447265625,
+      "rewards//std": 0.019817376509308815,
+      "step": 356
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0714,
+      "grad_norm": 2.472399950027466,
+      "kl": 0.14893962908536196,
+      "learning_rate": 9.906004562928863e-07,
+      "loss": 0.006,
+      "num_tokens": 3086656.0,
+      "reward": 0.7430419921875,
+      "reward_std": 0.008085759356617928,
+      "rewards//mean": 0.7430419921875,
+      "rewards//std": 0.017846059054136276,
+      "step": 357
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0716,
+      "grad_norm": 2.5224549770355225,
+      "kl": 0.16191406780853868,
+      "learning_rate": 9.905391157782897e-07,
+      "loss": 0.0065,
+      "num_tokens": 3095184.0,
+      "reward": 0.74932861328125,
+      "reward_std": 0.011135554872453213,
+      "rewards//mean": 0.74932861328125,
+      "rewards//std": 0.03566278889775276,
+      "step": 358
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0718,
+      "grad_norm": 2.772547483444214,
+      "kl": 0.17021930869668722,
+      "learning_rate": 9.904775776745956e-07,
+      "loss": 0.0068,
+      "num_tokens": 3103768.0,
+      "reward": 0.79095458984375,
+      "reward_std": 0.008317381143569946,
+      "rewards//mean": 0.79095458984375,
+      "rewards//std": 0.02636510692536831,
+      "step": 359
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.072,
+      "grad_norm": 2.4869027137756348,
+      "kl": 0.16306456178426743,
+      "learning_rate": 9.904158420065922e-07,
+      "loss": 0.0065,
+      "num_tokens": 3112464.0,
+      "reward": 0.76519775390625,
+      "reward_std": 0.006278190761804581,
+      "rewards//mean": 0.76519775390625,
+      "rewards//std": 0.03004978597164154,
+      "step": 360
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0722,
+      "grad_norm": 2.7342638969421387,
+      "kl": 0.16009332658722997,
+      "learning_rate": 9.903539087991461e-07,
+      "loss": 0.0064,
+      "num_tokens": 3121000.0,
+      "reward": 0.72418212890625,
+      "reward_std": 0.008755840361118317,
+      "rewards//mean": 0.72418212890625,
+      "rewards//std": 0.03662150725722313,
+      "step": 361
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0724,
+      "grad_norm": 2.810922622680664,
+      "kl": 0.1547168125398457,
+      "learning_rate": 9.902917780772042e-07,
+      "loss": 0.0062,
+      "num_tokens": 3129608.0,
+      "reward": 0.77642822265625,
+      "reward_std": 0.01392364501953125,
+      "rewards//mean": 0.77642822265625,
+      "rewards//std": 0.023206347599625587,
+      "step": 362
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0726,
+      "grad_norm": 2.8092007637023926,
+      "kl": 0.16291858348995447,
+      "learning_rate": 9.902294498657929e-07,
+      "loss": 0.0065,
+      "num_tokens": 3138360.0,
+      "reward": 0.7508544921875,
+      "reward_std": 0.009465081617236137,
+      "rewards//mean": 0.7508544921875,
+      "rewards//std": 0.04253233224153519,
+      "step": 363
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0728,
+      "grad_norm": 2.507169723510742,
+      "kl": 0.16634537931531668,
+      "learning_rate": 9.901669241900176e-07,
+      "loss": 0.0067,
+      "num_tokens": 3146912.0,
+      "reward": 0.73480224609375,
+      "reward_std": 0.007271946407854557,
+      "rewards//mean": 0.73480224609375,
+      "rewards//std": 0.02672211267054081,
+      "step": 364
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.073,
+      "grad_norm": 2.7649011611938477,
+      "kl": 0.15464997477829456,
+      "learning_rate": 9.90104201075064e-07,
+      "loss": 0.0062,
+      "num_tokens": 3155568.0,
+      "reward": 0.71844482421875,
+      "reward_std": 0.008541805669665337,
+      "rewards//mean": 0.71844482421875,
+      "rewards//std": 0.03847087174654007,
+      "step": 365
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0732,
+      "grad_norm": 2.8290657997131348,
+      "kl": 0.19138141721487045,
+      "learning_rate": 9.900412805461966e-07,
+      "loss": 0.0077,
+      "num_tokens": 3164336.0,
+      "reward": 0.76190185546875,
+      "reward_std": 0.007410317659378052,
+      "rewards//mean": 0.76190185546875,
+      "rewards//std": 0.036752309650182724,
+      "step": 366
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0734,
+      "grad_norm": 2.7856574058532715,
+      "kl": 0.15830178558826447,
+      "learning_rate": 9.899781626287602e-07,
+      "loss": 0.0063,
+      "num_tokens": 3173144.0,
+      "reward": 0.760009765625,
+      "reward_std": 0.008688227273523808,
+      "rewards//mean": 0.760009765625,
+      "rewards//std": 0.02757638320326805,
+      "step": 367
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0736,
+      "grad_norm": 2.695427894592285,
+      "kl": 0.17311879340559244,
+      "learning_rate": 9.899148473481784e-07,
+      "loss": 0.0069,
+      "num_tokens": 3181768.0,
+      "reward": 0.73980712890625,
+      "reward_std": 0.006761234253644943,
+      "rewards//mean": 0.73980712890625,
+      "rewards//std": 0.02978668175637722,
+      "step": 368
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0738,
+      "grad_norm": 2.656928062438965,
+      "kl": 0.1619602907449007,
+      "learning_rate": 9.898513347299547e-07,
+      "loss": 0.0065,
+      "num_tokens": 3190400.0,
+      "reward": 0.714599609375,
+      "reward_std": 0.007395018357783556,
+      "rewards//mean": 0.714599609375,
+      "rewards//std": 0.04246055334806442,
+      "step": 369
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.074,
+      "grad_norm": 2.7612054347991943,
+      "kl": 0.16792897693812847,
+      "learning_rate": 9.89787624799672e-07,
+      "loss": 0.0067,
+      "num_tokens": 3199032.0,
+      "reward": 0.77362060546875,
+      "reward_std": 0.009967046789824963,
+      "rewards//mean": 0.77362060546875,
+      "rewards//std": 0.029193807393312454,
+      "step": 370
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0742,
+      "grad_norm": 2.7488813400268555,
+      "kl": 0.17005004733800888,
+      "learning_rate": 9.897237175829926e-07,
+      "loss": 0.0068,
+      "num_tokens": 3207600.0,
+      "reward": 0.75494384765625,
+      "reward_std": 0.009035948663949966,
+      "rewards//mean": 0.75494384765625,
+      "rewards//std": 0.031119847670197487,
+      "step": 371
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0744,
+      "grad_norm": 2.553189754486084,
+      "kl": 0.16994115710258484,
+      "learning_rate": 9.896596131056582e-07,
+      "loss": 0.0068,
+      "num_tokens": 3216240.0,
+      "reward": 0.74267578125,
+      "reward_std": 0.008026436902582645,
+      "rewards//mean": 0.74267578125,
+      "rewards//std": 0.03289256617426872,
+      "step": 372
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0746,
+      "grad_norm": 2.361967086791992,
+      "kl": 0.1540005928836763,
+      "learning_rate": 9.895953113934903e-07,
+      "loss": 0.0062,
+      "num_tokens": 3224856.0,
+      "reward": 0.8001708984375,
+      "reward_std": 0.007031290326267481,
+      "rewards//mean": 0.8001708984375,
+      "rewards//std": 0.022404255345463753,
+      "step": 373
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0748,
+      "grad_norm": 2.7153866291046143,
+      "kl": 0.15861506760120392,
+      "learning_rate": 9.895308124723896e-07,
+      "loss": 0.0063,
+      "num_tokens": 3233608.0,
+      "reward": 0.7701416015625,
+      "reward_std": 0.01336634624749422,
+      "rewards//mean": 0.7701416015625,
+      "rewards//std": 0.03284627944231033,
+      "step": 374
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.075,
+      "grad_norm": 2.4990200996398926,
+      "kl": 0.1461920291185379,
+      "learning_rate": 9.89466116368336e-07,
+      "loss": 0.0058,
+      "num_tokens": 3242224.0,
+      "reward": 0.72381591796875,
+      "reward_std": 0.009897821582853794,
+      "rewards//mean": 0.72381591796875,
+      "rewards//std": 0.043557584285736084,
+      "step": 375
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0752,
+      "grad_norm": 2.413890838623047,
+      "kl": 0.15592028107494116,
+      "learning_rate": 9.894012231073895e-07,
+      "loss": 0.0062,
+      "num_tokens": 3250880.0,
+      "reward": 0.75830078125,
+      "reward_std": 0.00871911458671093,
+      "rewards//mean": 0.75830078125,
+      "rewards//std": 0.03690403327345848,
+      "step": 376
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0754,
+      "grad_norm": 2.9090678691864014,
+      "kl": 0.14931943733245134,
+      "learning_rate": 9.893361327156884e-07,
+      "loss": 0.006,
+      "num_tokens": 3259592.0,
+      "reward": 0.7562255859375,
+      "reward_std": 0.00859127752482891,
+      "rewards//mean": 0.7562255859375,
+      "rewards//std": 0.03542540967464447,
+      "step": 377
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0756,
+      "grad_norm": 2.6068999767303467,
+      "kl": 0.16098628798499703,
+      "learning_rate": 9.89270845219452e-07,
+      "loss": 0.0064,
+      "num_tokens": 3268216.0,
+      "reward": 0.7706298828125,
+      "reward_std": 0.013982309028506279,
+      "rewards//mean": 0.7706298828125,
+      "rewards//std": 0.03481682017445564,
+      "step": 378
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0758,
+      "grad_norm": 2.4730560779571533,
+      "kl": 0.15955154318362474,
+      "learning_rate": 9.892053606449774e-07,
+      "loss": 0.0064,
+      "num_tokens": 3276832.0,
+      "reward": 0.77764892578125,
+      "reward_std": 0.006985072512179613,
+      "rewards//mean": 0.77764892578125,
+      "rewards//std": 0.02482563443481922,
+      "step": 379
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.076,
+      "grad_norm": 2.840237855911255,
+      "kl": 0.16714267572388053,
+      "learning_rate": 9.891396790186422e-07,
+      "loss": 0.0067,
+      "num_tokens": 3285440.0,
+      "reward": 0.73150634765625,
+      "reward_std": 0.006967043504118919,
+      "rewards//mean": 0.73150634765625,
+      "rewards//std": 0.03609272837638855,
+      "step": 380
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0762,
+      "grad_norm": 2.4098997116088867,
+      "kl": 0.16293874941766262,
+      "learning_rate": 9.890738003669027e-07,
+      "loss": 0.0065,
+      "num_tokens": 3294056.0,
+      "reward": 0.77288818359375,
+      "reward_std": 0.00870993360877037,
+      "rewards//mean": 0.77288818359375,
+      "rewards//std": 0.04149697721004486,
+      "step": 381
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0764,
+      "grad_norm": 2.6688671112060547,
+      "kl": 0.16645270492881536,
+      "learning_rate": 9.89007724716295e-07,
+      "loss": 0.0067,
+      "num_tokens": 3302832.0,
+      "reward": 0.75201416015625,
+      "reward_std": 0.006812462583184242,
+      "rewards//mean": 0.75201416015625,
+      "rewards//std": 0.03431599214673042,
+      "step": 382
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0766,
+      "grad_norm": 2.900796890258789,
+      "kl": 0.1770854638889432,
+      "learning_rate": 9.889414520934343e-07,
+      "loss": 0.0071,
+      "num_tokens": 3311488.0,
+      "reward": 0.757080078125,
+      "reward_std": 0.009493831545114517,
+      "rewards//mean": 0.757080078125,
+      "rewards//std": 0.026105530560016632,
+      "step": 383
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0768,
+      "grad_norm": 2.7237555980682373,
+      "kl": 0.16892468417063355,
+      "learning_rate": 9.88874982525015e-07,
+      "loss": 0.0068,
+      "num_tokens": 3320128.0,
+      "reward": 0.72174072265625,
+      "reward_std": 0.00853054877370596,
+      "rewards//mean": 0.72174072265625,
+      "rewards//std": 0.039785850793123245,
+      "step": 384
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.077,
+      "grad_norm": 2.570565938949585,
+      "kl": 0.17061032820492983,
+      "learning_rate": 9.888083160378112e-07,
+      "loss": 0.0068,
+      "num_tokens": 3328744.0,
+      "reward": 0.7491455078125,
+      "reward_std": 0.008467968553304672,
+      "rewards//mean": 0.7491455078125,
+      "rewards//std": 0.02584063820540905,
+      "step": 385
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0772,
+      "grad_norm": 2.565676689147949,
+      "kl": 0.15762492455542088,
+      "learning_rate": 9.887414526586763e-07,
+      "loss": 0.0063,
+      "num_tokens": 3337352.0,
+      "reward": 0.746826171875,
+      "reward_std": 0.00936988927423954,
+      "rewards//mean": 0.746826171875,
+      "rewards//std": 0.030208613723516464,
+      "step": 386
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0774,
+      "grad_norm": 2.5404958724975586,
+      "kl": 0.1645594099536538,
+      "learning_rate": 9.886743924145426e-07,
+      "loss": 0.0066,
+      "num_tokens": 3345952.0,
+      "reward": 0.78057861328125,
+      "reward_std": 0.007525625638663769,
+      "rewards//mean": 0.78057861328125,
+      "rewards//std": 0.029721051454544067,
+      "step": 387
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0776,
+      "grad_norm": 2.538297414779663,
+      "kl": 0.16444916930049658,
+      "learning_rate": 9.886071353324222e-07,
+      "loss": 0.0066,
+      "num_tokens": 3354552.0,
+      "reward": 0.73828125,
+      "reward_std": 0.009832067415118217,
+      "rewards//mean": 0.73828125,
+      "rewards//std": 0.028039097785949707,
+      "step": 388
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0778,
+      "grad_norm": 2.6103405952453613,
+      "kl": 0.17876138631254435,
+      "learning_rate": 9.88539681439406e-07,
+      "loss": 0.0072,
+      "num_tokens": 3363192.0,
+      "reward": 0.74212646484375,
+      "reward_std": 0.009619161486625671,
+      "rewards//mean": 0.74212646484375,
+      "rewards//std": 0.03128479793667793,
+      "step": 389
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.078,
+      "grad_norm": 2.5139098167419434,
+      "kl": 0.1818069638684392,
+      "learning_rate": 9.884720307626646e-07,
+      "loss": 0.0073,
+      "num_tokens": 3371832.0,
+      "reward": 0.75091552734375,
+      "reward_std": 0.008223002776503563,
+      "rewards//mean": 0.75091552734375,
+      "rewards//std": 0.032842766493558884,
+      "step": 390
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0782,
+      "grad_norm": 2.67838978767395,
+      "kl": 0.15979084372520447,
+      "learning_rate": 9.884041833294475e-07,
+      "loss": 0.0064,
+      "num_tokens": 3380448.0,
+      "reward": 0.774658203125,
+      "reward_std": 0.010571452789008617,
+      "rewards//mean": 0.774658203125,
+      "rewards//std": 0.024899380281567574,
+      "step": 391
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0784,
+      "grad_norm": 2.609266757965088,
+      "kl": 0.17010739585384727,
+      "learning_rate": 9.883361391670839e-07,
+      "loss": 0.0068,
+      "num_tokens": 3389040.0,
+      "reward": 0.75555419921875,
+      "reward_std": 0.010208028368651867,
+      "rewards//mean": 0.75555419921875,
+      "rewards//std": 0.02784671261906624,
+      "step": 392
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0786,
+      "grad_norm": 2.705087184906006,
+      "kl": 0.1755635621957481,
+      "learning_rate": 9.882678983029817e-07,
+      "loss": 0.007,
+      "num_tokens": 3397736.0,
+      "reward": 0.748291015625,
+      "reward_std": 0.00819418951869011,
+      "rewards//mean": 0.748291015625,
+      "rewards//std": 0.03141137585043907,
+      "step": 393
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0788,
+      "grad_norm": 2.8524513244628906,
+      "kl": 0.17735581519082189,
+      "learning_rate": 9.881994607646286e-07,
+      "loss": 0.0071,
+      "num_tokens": 3406328.0,
+      "reward": 0.7484130859375,
+      "reward_std": 0.009125860407948494,
+      "rewards//mean": 0.7484130859375,
+      "rewards//std": 0.011681877076625824,
+      "step": 394
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.079,
+      "grad_norm": 2.815516948699951,
+      "kl": 0.17463888693600893,
+      "learning_rate": 9.881308265795911e-07,
+      "loss": 0.007,
+      "num_tokens": 3414960.0,
+      "reward": 0.73016357421875,
+      "reward_std": 0.00860263966023922,
+      "rewards//mean": 0.73016357421875,
+      "rewards//std": 0.034648314118385315,
+      "step": 395
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0792,
+      "grad_norm": 3.0293967723846436,
+      "kl": 0.18000991456210613,
+      "learning_rate": 9.88061995775515e-07,
+      "loss": 0.0072,
+      "num_tokens": 3423696.0,
+      "reward": 0.720458984375,
+      "reward_std": 0.007856153883039951,
+      "rewards//mean": 0.720458984375,
+      "rewards//std": 0.03832308202981949,
+      "step": 396
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0794,
+      "grad_norm": 2.5878942012786865,
+      "kl": 0.18182779476046562,
+      "learning_rate": 9.879929683801253e-07,
+      "loss": 0.0073,
+      "num_tokens": 3432248.0,
+      "reward": 0.75244140625,
+      "reward_std": 0.007938584312796593,
+      "rewards//mean": 0.75244140625,
+      "rewards//std": 0.02669837884604931,
+      "step": 397
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0796,
+      "grad_norm": 2.4721519947052,
+      "kl": 0.16829326748847961,
+      "learning_rate": 9.879237444212264e-07,
+      "loss": 0.0067,
+      "num_tokens": 3440928.0,
+      "reward": 0.76171875,
+      "reward_std": 0.008632577955722809,
+      "rewards//mean": 0.76171875,
+      "rewards//std": 0.01977158710360527,
+      "step": 398
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0798,
+      "grad_norm": 2.558156967163086,
+      "kl": 0.17831025645136833,
+      "learning_rate": 9.878543239267014e-07,
+      "loss": 0.0071,
+      "num_tokens": 3449560.0,
+      "reward": 0.75030517578125,
+      "reward_std": 0.007332679349929094,
+      "rewards//mean": 0.75030517578125,
+      "rewards//std": 0.023222649469971657,
+      "step": 399
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.08,
+      "grad_norm": 2.3982725143432617,
+      "kl": 0.1726833526045084,
+      "learning_rate": 9.877847069245133e-07,
+      "loss": 0.0069,
+      "num_tokens": 3458264.0,
+      "reward": 0.7664794921875,
+      "reward_std": 0.0074759190902113914,
+      "rewards//mean": 0.7664794921875,
+      "rewards//std": 0.023963475599884987,
+      "step": 400
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0802,
+      "grad_norm": 2.6396090984344482,
+      "kl": 0.18833619728684425,
+      "learning_rate": 9.877148934427035e-07,
+      "loss": 0.0075,
+      "num_tokens": 3466896.0,
+      "reward": 0.7677001953125,
+      "reward_std": 0.006921134889125824,
+      "rewards//mean": 0.7677001953125,
+      "rewards//std": 0.029165079817175865,
+      "step": 401
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0804,
+      "grad_norm": 2.3063488006591797,
+      "kl": 0.18033870682120323,
+      "learning_rate": 9.876448835093929e-07,
+      "loss": 0.0072,
+      "num_tokens": 3475584.0,
+      "reward": 0.76837158203125,
+      "reward_std": 0.00596450874581933,
+      "rewards//mean": 0.76837158203125,
+      "rewards//std": 0.033730026334524155,
+      "step": 402
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0806,
+      "grad_norm": 2.6989567279815674,
+      "kl": 0.19588027335703373,
+      "learning_rate": 9.875746771527815e-07,
+      "loss": 0.0078,
+      "num_tokens": 3484184.0,
+      "reward": 0.77716064453125,
+      "reward_std": 0.008181614801287651,
+      "rewards//mean": 0.77716064453125,
+      "rewards//std": 0.02728992886841297,
+      "step": 403
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0808,
+      "grad_norm": 2.426013469696045,
+      "kl": 0.18516316264867783,
+      "learning_rate": 9.875042744011486e-07,
+      "loss": 0.0074,
+      "num_tokens": 3492760.0,
+      "reward": 0.76885986328125,
+      "reward_std": 0.007055317983031273,
+      "rewards//mean": 0.76885986328125,
+      "rewards//std": 0.017928466200828552,
+      "step": 404
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.081,
+      "grad_norm": 2.4702301025390625,
+      "kl": 0.1746185226365924,
+      "learning_rate": 9.874336752828522e-07,
+      "loss": 0.007,
+      "num_tokens": 3501344.0,
+      "reward": 0.76171875,
+      "reward_std": 0.010278567671775818,
+      "rewards//mean": 0.76171875,
+      "rewards//std": 0.0270722433924675,
+      "step": 405
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0812,
+      "grad_norm": 2.7907369136810303,
+      "kl": 0.18580629862844944,
+      "learning_rate": 9.873628798263295e-07,
+      "loss": 0.0074,
+      "num_tokens": 3510008.0,
+      "reward": 0.7650146484375,
+      "reward_std": 0.006237374618649483,
+      "rewards//mean": 0.7650146484375,
+      "rewards//std": 0.034500602632761,
+      "step": 406
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0814,
+      "grad_norm": 2.4231181144714355,
+      "kl": 0.18027489073574543,
+      "learning_rate": 9.872918880600973e-07,
+      "loss": 0.0072,
+      "num_tokens": 3518752.0,
+      "reward": 0.764892578125,
+      "reward_std": 0.005956288427114487,
+      "rewards//mean": 0.764892578125,
+      "rewards//std": 0.016305776312947273,
+      "step": 407
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0816,
+      "grad_norm": 2.4755072593688965,
+      "kl": 0.19561395607888699,
+      "learning_rate": 9.87220700012751e-07,
+      "loss": 0.0078,
+      "num_tokens": 3527400.0,
+      "reward": 0.762939453125,
+      "reward_std": 0.006552073638886213,
+      "rewards//mean": 0.762939453125,
+      "rewards//std": 0.036029815673828125,
+      "step": 408
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0818,
+      "grad_norm": 2.598135232925415,
+      "kl": 0.18279425613582134,
+      "learning_rate": 9.871493157129647e-07,
+      "loss": 0.0073,
+      "num_tokens": 3536064.0,
+      "reward": 0.74993896484375,
+      "reward_std": 0.008556658402085304,
+      "rewards//mean": 0.74993896484375,
+      "rewards//std": 0.02316194958984852,
+      "step": 409
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.082,
+      "grad_norm": 2.7481443881988525,
+      "kl": 0.18953238148242235,
+      "learning_rate": 9.870777351894926e-07,
+      "loss": 0.0076,
+      "num_tokens": 3544736.0,
+      "reward": 0.71820068359375,
+      "reward_std": 0.008066165260970592,
+      "rewards//mean": 0.71820068359375,
+      "rewards//std": 0.028888877481222153,
+      "step": 410
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0822,
+      "grad_norm": 2.787365198135376,
+      "kl": 0.1878298930823803,
+      "learning_rate": 9.870059584711668e-07,
+      "loss": 0.0075,
+      "num_tokens": 3553424.0,
+      "reward": 0.74188232421875,
+      "reward_std": 0.007019890006631613,
+      "rewards//mean": 0.74188232421875,
+      "rewards//std": 0.02889254502952099,
+      "step": 411
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0824,
+      "grad_norm": 2.81783390045166,
+      "kl": 0.19163534324616194,
+      "learning_rate": 9.869339855868991e-07,
+      "loss": 0.0077,
+      "num_tokens": 3562040.0,
+      "reward": 0.7288818359375,
+      "reward_std": 0.009509474039077759,
+      "rewards//mean": 0.7288818359375,
+      "rewards//std": 0.03359359875321388,
+      "step": 412
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0826,
+      "grad_norm": 2.587827205657959,
+      "kl": 0.2044460643082857,
+      "learning_rate": 9.868618165656804e-07,
+      "loss": 0.0082,
+      "num_tokens": 3570696.0,
+      "reward": 0.74566650390625,
+      "reward_std": 0.008736889809370041,
+      "rewards//mean": 0.74566650390625,
+      "rewards//std": 0.03351707383990288,
+      "step": 413
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0828,
+      "grad_norm": 2.6200897693634033,
+      "kl": 0.18804363813251257,
+      "learning_rate": 9.8678945143658e-07,
+      "loss": 0.0075,
+      "num_tokens": 3579256.0,
+      "reward": 0.7288818359375,
+      "reward_std": 0.009151293896138668,
+      "rewards//mean": 0.7288818359375,
+      "rewards//std": 0.03662879392504692,
+      "step": 414
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.083,
+      "grad_norm": 2.506772994995117,
+      "kl": 0.1931857457384467,
+      "learning_rate": 9.86716890228747e-07,
+      "loss": 0.0077,
+      "num_tokens": 3587928.0,
+      "reward": 0.72314453125,
+      "reward_std": 0.006545564159750938,
+      "rewards//mean": 0.72314453125,
+      "rewards//std": 0.03633522614836693,
+      "step": 415
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0832,
+      "grad_norm": 2.445132255554199,
+      "kl": 0.20388566982001066,
+      "learning_rate": 9.866441329714087e-07,
+      "loss": 0.0082,
+      "num_tokens": 3596568.0,
+      "reward": 0.761474609375,
+      "reward_std": 0.005879071541130543,
+      "rewards//mean": 0.761474609375,
+      "rewards//std": 0.020397162064909935,
+      "step": 416
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0834,
+      "grad_norm": 2.6903209686279297,
+      "kl": 0.20185576286166906,
+      "learning_rate": 9.86571179693872e-07,
+      "loss": 0.0081,
+      "num_tokens": 3605248.0,
+      "reward": 0.78009033203125,
+      "reward_std": 0.010339450091123581,
+      "rewards//mean": 0.78009033203125,
+      "rewards//std": 0.026325460523366928,
+      "step": 417
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0836,
+      "grad_norm": 2.6082499027252197,
+      "kl": 0.1881139650940895,
+      "learning_rate": 9.86498030425522e-07,
+      "loss": 0.0075,
+      "num_tokens": 3613840.0,
+      "reward": 0.753173828125,
+      "reward_std": 0.009717400185763836,
+      "rewards//mean": 0.753173828125,
+      "rewards//std": 0.033479172736406326,
+      "step": 418
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0838,
+      "grad_norm": 2.484720230102539,
+      "kl": 0.19344091974198818,
+      "learning_rate": 9.864246851958237e-07,
+      "loss": 0.0077,
+      "num_tokens": 3622464.0,
+      "reward": 0.75836181640625,
+      "reward_std": 0.010392667725682259,
+      "rewards//mean": 0.75836181640625,
+      "rewards//std": 0.04072520136833191,
+      "step": 419
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.084,
+      "grad_norm": 2.8422417640686035,
+      "kl": 0.18848030921071768,
+      "learning_rate": 9.863511440343205e-07,
+      "loss": 0.0075,
+      "num_tokens": 3631032.0,
+      "reward": 0.74420166015625,
+      "reward_std": 0.007764378562569618,
+      "rewards//mean": 0.74420166015625,
+      "rewards//std": 0.029569394886493683,
+      "step": 420
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0842,
+      "grad_norm": 2.5308332443237305,
+      "kl": 0.1926432317122817,
+      "learning_rate": 9.862774069706345e-07,
+      "loss": 0.0077,
+      "num_tokens": 3639672.0,
+      "reward": 0.74664306640625,
+      "reward_std": 0.0067763011902570724,
+      "rewards//mean": 0.74664306640625,
+      "rewards//std": 0.030432218685746193,
+      "step": 421
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0844,
+      "grad_norm": 2.7227256298065186,
+      "kl": 0.22925530839711428,
+      "learning_rate": 9.862034740344671e-07,
+      "loss": 0.0092,
+      "num_tokens": 3648336.0,
+      "reward": 0.771484375,
+      "reward_std": 0.008542067371308804,
+      "rewards//mean": 0.771484375,
+      "rewards//std": 0.023302771151065826,
+      "step": 422
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0846,
+      "grad_norm": 2.769712448120117,
+      "kl": 0.19543266110122204,
+      "learning_rate": 9.861293452555986e-07,
+      "loss": 0.0078,
+      "num_tokens": 3656896.0,
+      "reward": 0.79376220703125,
+      "reward_std": 0.007219559047371149,
+      "rewards//mean": 0.79376220703125,
+      "rewards//std": 0.03004222735762596,
+      "step": 423
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0848,
+      "grad_norm": 2.576681613922119,
+      "kl": 0.19337232876569033,
+      "learning_rate": 9.86055020663888e-07,
+      "loss": 0.0077,
+      "num_tokens": 3665504.0,
+      "reward": 0.733154296875,
+      "reward_std": 0.006978962570428848,
+      "rewards//mean": 0.733154296875,
+      "rewards//std": 0.02180885523557663,
+      "step": 424
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.085,
+      "grad_norm": 2.465479850769043,
+      "kl": 0.1821321528404951,
+      "learning_rate": 9.859805002892731e-07,
+      "loss": 0.0073,
+      "num_tokens": 3674080.0,
+      "reward": 0.73333740234375,
+      "reward_std": 0.007003418169915676,
+      "rewards//mean": 0.73333740234375,
+      "rewards//std": 0.03618655353784561,
+      "step": 425
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0852,
+      "grad_norm": 2.6620612144470215,
+      "kl": 0.20288071408867836,
+      "learning_rate": 9.859057841617708e-07,
+      "loss": 0.0081,
+      "num_tokens": 3682640.0,
+      "reward": 0.7447509765625,
+      "reward_std": 0.008478684350848198,
+      "rewards//mean": 0.7447509765625,
+      "rewards//std": 0.03240830451250076,
+      "step": 426
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0854,
+      "grad_norm": 2.6503520011901855,
+      "kl": 0.19767640065401793,
+      "learning_rate": 9.858308723114768e-07,
+      "loss": 0.0079,
+      "num_tokens": 3691304.0,
+      "reward": 0.78692626953125,
+      "reward_std": 0.008378500118851662,
+      "rewards//mean": 0.78692626953125,
+      "rewards//std": 0.027062665671110153,
+      "step": 427
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0856,
+      "grad_norm": 2.59274959564209,
+      "kl": 0.20678219757974148,
+      "learning_rate": 9.857557647685655e-07,
+      "loss": 0.0083,
+      "num_tokens": 3700024.0,
+      "reward": 0.75311279296875,
+      "reward_std": 0.00883854366838932,
+      "rewards//mean": 0.75311279296875,
+      "rewards//std": 0.03340170904994011,
+      "step": 428
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0858,
+      "grad_norm": 2.514921188354492,
+      "kl": 0.1842403383925557,
+      "learning_rate": 9.856804615632901e-07,
+      "loss": 0.0074,
+      "num_tokens": 3708784.0,
+      "reward": 0.75360107421875,
+      "reward_std": 0.009444896131753922,
+      "rewards//mean": 0.75360107421875,
+      "rewards//std": 0.029522772878408432,
+      "step": 429
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.086,
+      "grad_norm": 2.777625799179077,
+      "kl": 0.19502401165664196,
+      "learning_rate": 9.856049627259832e-07,
+      "loss": 0.0078,
+      "num_tokens": 3717392.0,
+      "reward": 0.76678466796875,
+      "reward_std": 0.007297373842447996,
+      "rewards//mean": 0.76678466796875,
+      "rewards//std": 0.02159462496638298,
+      "step": 430
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0862,
+      "grad_norm": 2.677000045776367,
+      "kl": 0.18325078207999468,
+      "learning_rate": 9.85529268287055e-07,
+      "loss": 0.0073,
+      "num_tokens": 3726008.0,
+      "reward": 0.74859619140625,
+      "reward_std": 0.007576840464025736,
+      "rewards//mean": 0.74859619140625,
+      "rewards//std": 0.02463035099208355,
+      "step": 431
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0864,
+      "grad_norm": 2.5007803440093994,
+      "kl": 0.19161915499716997,
+      "learning_rate": 9.854533782769959e-07,
+      "loss": 0.0077,
+      "num_tokens": 3734584.0,
+      "reward": 0.74884033203125,
+      "reward_std": 0.007008885033428669,
+      "rewards//mean": 0.74884033203125,
+      "rewards//std": 0.02085285261273384,
+      "step": 432
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0866,
+      "grad_norm": 2.587076187133789,
+      "kl": 0.2080300049856305,
+      "learning_rate": 9.853772927263739e-07,
+      "loss": 0.0083,
+      "num_tokens": 3743296.0,
+      "reward": 0.7801513671875,
+      "reward_std": 0.007318174000829458,
+      "rewards//mean": 0.7801513671875,
+      "rewards//std": 0.03165404871106148,
+      "step": 433
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0868,
+      "grad_norm": 2.896596908569336,
+      "kl": 0.1951505597680807,
+      "learning_rate": 9.853010116658366e-07,
+      "loss": 0.0078,
+      "num_tokens": 3752000.0,
+      "reward": 0.7440185546875,
+      "reward_std": 0.00882251188158989,
+      "rewards//mean": 0.7440185546875,
+      "rewards//std": 0.02630971372127533,
+      "step": 434
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.087,
+      "grad_norm": 2.714966297149658,
+      "kl": 0.19744882453233004,
+      "learning_rate": 9.852245351261097e-07,
+      "loss": 0.0079,
+      "num_tokens": 3760584.0,
+      "reward": 0.71649169921875,
+      "reward_std": 0.009333435446023941,
+      "rewards//mean": 0.71649169921875,
+      "rewards//std": 0.04489992558956146,
+      "step": 435
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0872,
+      "grad_norm": 2.624218702316284,
+      "kl": 0.19562861416488886,
+      "learning_rate": 9.851478631379982e-07,
+      "loss": 0.0078,
+      "num_tokens": 3769168.0,
+      "reward": 0.742919921875,
+      "reward_std": 0.008637506514787674,
+      "rewards//mean": 0.742919921875,
+      "rewards//std": 0.03229343146085739,
+      "step": 436
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0874,
+      "grad_norm": 2.5103540420532227,
+      "kl": 0.19844850152730942,
+      "learning_rate": 9.850709957323854e-07,
+      "loss": 0.0079,
+      "num_tokens": 3777808.0,
+      "reward": 0.74249267578125,
+      "reward_std": 0.007015228737145662,
+      "rewards//mean": 0.74249267578125,
+      "rewards//std": 0.02400221675634384,
+      "step": 437
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0876,
+      "grad_norm": 2.376951217651367,
+      "kl": 0.19819010514765978,
+      "learning_rate": 9.849939329402336e-07,
+      "loss": 0.0079,
+      "num_tokens": 3786392.0,
+      "reward": 0.7506103515625,
+      "reward_std": 0.007007762789726257,
+      "rewards//mean": 0.7506103515625,
+      "rewards//std": 0.019626734778285027,
+      "step": 438
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0878,
+      "grad_norm": 2.7214441299438477,
+      "kl": 0.20057359337806702,
+      "learning_rate": 9.849166747925834e-07,
+      "loss": 0.008,
+      "num_tokens": 3795064.0,
+      "reward": 0.77362060546875,
+      "reward_std": 0.008020205423235893,
+      "rewards//mean": 0.77362060546875,
+      "rewards//std": 0.02572754956781864,
+      "step": 439
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.088,
+      "grad_norm": 2.4217584133148193,
+      "kl": 0.20243626087903976,
+      "learning_rate": 9.848392213205547e-07,
+      "loss": 0.0081,
+      "num_tokens": 3803632.0,
+      "reward": 0.7454833984375,
+      "reward_std": 0.007380123715847731,
+      "rewards//mean": 0.7454833984375,
+      "rewards//std": 0.040875695645809174,
+      "step": 440
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0882,
+      "grad_norm": 2.661808729171753,
+      "kl": 0.18222533399239182,
+      "learning_rate": 9.847615725553455e-07,
+      "loss": 0.0073,
+      "num_tokens": 3812296.0,
+      "reward": 0.7913818359375,
+      "reward_std": 0.008068419992923737,
+      "rewards//mean": 0.7913818359375,
+      "rewards//std": 0.022720929235219955,
+      "step": 441
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0884,
+      "grad_norm": 2.587676525115967,
+      "kl": 0.191447539255023,
+      "learning_rate": 9.84683728528233e-07,
+      "loss": 0.0077,
+      "num_tokens": 3820952.0,
+      "reward": 0.7510986328125,
+      "reward_std": 0.009070787578821182,
+      "rewards//mean": 0.7510986328125,
+      "rewards//std": 0.0328776054084301,
+      "step": 442
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0886,
+      "grad_norm": 2.9863452911376953,
+      "kl": 0.20910645369440317,
+      "learning_rate": 9.846056892705727e-07,
+      "loss": 0.0084,
+      "num_tokens": 3829712.0,
+      "reward": 0.71697998046875,
+      "reward_std": 0.008707192726433277,
+      "rewards//mean": 0.71697998046875,
+      "rewards//std": 0.032932523638010025,
+      "step": 443
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0888,
+      "grad_norm": 2.7540276050567627,
+      "kl": 0.2015506774187088,
+      "learning_rate": 9.845274548137985e-07,
+      "loss": 0.0081,
+      "num_tokens": 3838296.0,
+      "reward": 0.73382568359375,
+      "reward_std": 0.007177338935434818,
+      "rewards//mean": 0.73382568359375,
+      "rewards//std": 0.012887472286820412,
+      "step": 444
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.089,
+      "grad_norm": 2.6551339626312256,
+      "kl": 0.2096218653023243,
+      "learning_rate": 9.844490251894236e-07,
+      "loss": 0.0084,
+      "num_tokens": 3846960.0,
+      "reward": 0.7542724609375,
+      "reward_std": 0.007436465937644243,
+      "rewards//mean": 0.7542724609375,
+      "rewards//std": 0.02651832066476345,
+      "step": 445
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0892,
+      "grad_norm": 2.52319073677063,
+      "kl": 0.19700186513364315,
+      "learning_rate": 9.843704004290392e-07,
+      "loss": 0.0079,
+      "num_tokens": 3855688.0,
+      "reward": 0.7493896484375,
+      "reward_std": 0.006621568463742733,
+      "rewards//mean": 0.7493896484375,
+      "rewards//std": 0.03030742146074772,
+      "step": 446
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0894,
+      "grad_norm": 2.7591910362243652,
+      "kl": 0.2017053822055459,
+      "learning_rate": 9.842915805643156e-07,
+      "loss": 0.0081,
+      "num_tokens": 3864312.0,
+      "reward": 0.7664794921875,
+      "reward_std": 0.008353838697075844,
+      "rewards//mean": 0.7664794921875,
+      "rewards//std": 0.023245209828019142,
+      "step": 447
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0896,
+      "grad_norm": 2.4190142154693604,
+      "kl": 0.1878850907087326,
+      "learning_rate": 9.84212565627001e-07,
+      "loss": 0.0075,
+      "num_tokens": 3872992.0,
+      "reward": 0.7568359375,
+      "reward_std": 0.009020679630339146,
+      "rewards//mean": 0.7568359375,
+      "rewards//std": 0.02271326445043087,
+      "step": 448
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0898,
+      "grad_norm": 2.5284149646759033,
+      "kl": 0.21311705745756626,
+      "learning_rate": 9.841333556489232e-07,
+      "loss": 0.0085,
+      "num_tokens": 3881712.0,
+      "reward": 0.76666259765625,
+      "reward_std": 0.006807057186961174,
+      "rewards//mean": 0.76666259765625,
+      "rewards//std": 0.027252739295363426,
+      "step": 449
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.09,
+      "grad_norm": 2.463273763656616,
+      "kl": 0.1890072450041771,
+      "learning_rate": 9.840539506619872e-07,
+      "loss": 0.0076,
+      "num_tokens": 3890312.0,
+      "reward": 0.74871826171875,
+      "reward_std": 0.0067598591558635235,
+      "rewards//mean": 0.74871826171875,
+      "rewards//std": 0.03121940791606903,
+      "step": 450
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0902,
+      "grad_norm": 2.689650774002075,
+      "kl": 0.20761975646018982,
+      "learning_rate": 9.83974350698178e-07,
+      "loss": 0.0083,
+      "num_tokens": 3899016.0,
+      "reward": 0.7344970703125,
+      "reward_std": 0.008867532014846802,
+      "rewards//mean": 0.7344970703125,
+      "rewards//std": 0.0433250293135643,
+      "step": 451
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0904,
+      "grad_norm": 2.30403733253479,
+      "kl": 0.2046855976805091,
+      "learning_rate": 9.838945557895584e-07,
+      "loss": 0.0082,
+      "num_tokens": 3907752.0,
+      "reward": 0.777587890625,
+      "reward_std": 0.007500022184103727,
+      "rewards//mean": 0.777587890625,
+      "rewards//std": 0.018582813441753387,
+      "step": 452
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0906,
+      "grad_norm": 2.616736650466919,
+      "kl": 0.21534956805408,
+      "learning_rate": 9.838145659682692e-07,
+      "loss": 0.0086,
+      "num_tokens": 3916496.0,
+      "reward": 0.7713623046875,
+      "reward_std": 0.008273258805274963,
+      "rewards//mean": 0.7713623046875,
+      "rewards//std": 0.03000018745660782,
+      "step": 453
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0908,
+      "grad_norm": 2.856215715408325,
+      "kl": 0.1973020858131349,
+      "learning_rate": 9.83734381266531e-07,
+      "loss": 0.0079,
+      "num_tokens": 3925008.0,
+      "reward": 0.73614501953125,
+      "reward_std": 0.011645911261439323,
+      "rewards//mean": 0.73614501953125,
+      "rewards//std": 0.03672264143824577,
+      "step": 454
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.091,
+      "grad_norm": 2.643582582473755,
+      "kl": 0.19773272797465324,
+      "learning_rate": 9.836540017166419e-07,
+      "loss": 0.0079,
+      "num_tokens": 3933696.0,
+      "reward": 0.73785400390625,
+      "reward_std": 0.0073912725783884525,
+      "rewards//mean": 0.73785400390625,
+      "rewards//std": 0.032764315605163574,
+      "step": 455
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0912,
+      "grad_norm": 2.69028902053833,
+      "kl": 0.2114551505073905,
+      "learning_rate": 9.835734273509785e-07,
+      "loss": 0.0085,
+      "num_tokens": 3942304.0,
+      "reward": 0.779541015625,
+      "reward_std": 0.006727224215865135,
+      "rewards//mean": 0.779541015625,
+      "rewards//std": 0.022128576412796974,
+      "step": 456
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0914,
+      "grad_norm": 2.5662038326263428,
+      "kl": 0.1863793758675456,
+      "learning_rate": 9.834926582019966e-07,
+      "loss": 0.0075,
+      "num_tokens": 3950968.0,
+      "reward": 0.7479248046875,
+      "reward_std": 0.007806117180734873,
+      "rewards//mean": 0.7479248046875,
+      "rewards//std": 0.02594820410013199,
+      "step": 457
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0916,
+      "grad_norm": 2.378325939178467,
+      "kl": 0.18959766253829002,
+      "learning_rate": 9.834116943022297e-07,
+      "loss": 0.0076,
+      "num_tokens": 3959640.0,
+      "reward": 0.74444580078125,
+      "reward_std": 0.007687430828809738,
+      "rewards//mean": 0.74444580078125,
+      "rewards//std": 0.032067108899354935,
+      "step": 458
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0918,
+      "grad_norm": 2.345531702041626,
+      "kl": 0.19898757711052895,
+      "learning_rate": 9.8333053568429e-07,
+      "loss": 0.008,
+      "num_tokens": 3968256.0,
+      "reward": 0.77471923828125,
+      "reward_std": 0.009729254059493542,
+      "rewards//mean": 0.77471923828125,
+      "rewards//std": 0.02229410409927368,
+      "step": 459
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.092,
+      "grad_norm": 2.5729498863220215,
+      "kl": 0.19167259149253368,
+      "learning_rate": 9.832491823808686e-07,
+      "loss": 0.0077,
+      "num_tokens": 3977024.0,
+      "reward": 0.77337646484375,
+      "reward_std": 0.008461561053991318,
+      "rewards//mean": 0.77337646484375,
+      "rewards//std": 0.02044006995856762,
+      "step": 460
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0922,
+      "grad_norm": 2.286104679107666,
+      "kl": 0.20634953398257494,
+      "learning_rate": 9.831676344247342e-07,
+      "loss": 0.0083,
+      "num_tokens": 3985664.0,
+      "reward": 0.72589111328125,
+      "reward_std": 0.00819789431989193,
+      "rewards//mean": 0.72589111328125,
+      "rewards//std": 0.03155507892370224,
+      "step": 461
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0924,
+      "grad_norm": 2.541599750518799,
+      "kl": 0.1981488335877657,
+      "learning_rate": 9.830858918487346e-07,
+      "loss": 0.0079,
+      "num_tokens": 3994480.0,
+      "reward": 0.760009765625,
+      "reward_std": 0.008511878550052643,
+      "rewards//mean": 0.760009765625,
+      "rewards//std": 0.02578815445303917,
+      "step": 462
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0926,
+      "grad_norm": 2.3062171936035156,
+      "kl": 0.18402158375829458,
+      "learning_rate": 9.830039546857952e-07,
+      "loss": 0.0074,
+      "num_tokens": 4003112.0,
+      "reward": 0.73883056640625,
+      "reward_std": 0.00948390644043684,
+      "rewards//mean": 0.73883056640625,
+      "rewards//std": 0.0378524586558342,
+      "step": 463
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0928,
+      "grad_norm": 2.393819808959961,
+      "kl": 0.2080188114196062,
+      "learning_rate": 9.829218229689209e-07,
+      "loss": 0.0083,
+      "num_tokens": 4011720.0,
+      "reward": 0.744384765625,
+      "reward_std": 0.008310135453939438,
+      "rewards//mean": 0.744384765625,
+      "rewards//std": 0.03354422003030777,
+      "step": 464
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.093,
+      "grad_norm": 2.644294500350952,
+      "kl": 0.20960933715105057,
+      "learning_rate": 9.828394967311938e-07,
+      "loss": 0.0084,
+      "num_tokens": 4020384.0,
+      "reward": 0.7265625,
+      "reward_std": 0.009460685774683952,
+      "rewards//mean": 0.7265625,
+      "rewards//std": 0.03722097724676132,
+      "step": 465
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0932,
+      "grad_norm": 2.4498705863952637,
+      "kl": 0.19817031733691692,
+      "learning_rate": 9.827569760057754e-07,
+      "loss": 0.0079,
+      "num_tokens": 4029016.0,
+      "reward": 0.76434326171875,
+      "reward_std": 0.010624798946082592,
+      "rewards//mean": 0.76434326171875,
+      "rewards//std": 0.020740758627653122,
+      "step": 466
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0934,
+      "grad_norm": 2.5488860607147217,
+      "kl": 0.18394710682332516,
+      "learning_rate": 9.826742608259047e-07,
+      "loss": 0.0074,
+      "num_tokens": 4037688.0,
+      "reward": 0.781982421875,
+      "reward_std": 0.010184954851865768,
+      "rewards//mean": 0.781982421875,
+      "rewards//std": 0.03223337233066559,
+      "step": 467
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0936,
+      "grad_norm": 2.507493495941162,
+      "kl": 0.19846187718212605,
+      "learning_rate": 9.825913512248995e-07,
+      "loss": 0.0079,
+      "num_tokens": 4046304.0,
+      "reward": 0.74517822265625,
+      "reward_std": 0.007261100225150585,
+      "rewards//mean": 0.74517822265625,
+      "rewards//std": 0.023946020752191544,
+      "step": 468
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0938,
+      "grad_norm": 2.748115301132202,
+      "kl": 0.19336541090160608,
+      "learning_rate": 9.825082472361556e-07,
+      "loss": 0.0077,
+      "num_tokens": 4054992.0,
+      "reward": 0.71734619140625,
+      "reward_std": 0.007476060185581446,
+      "rewards//mean": 0.71734619140625,
+      "rewards//std": 0.03136115521192551,
+      "step": 469
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.094,
+      "grad_norm": 2.452084541320801,
+      "kl": 0.20964460261166096,
+      "learning_rate": 9.824249488931475e-07,
+      "loss": 0.0084,
+      "num_tokens": 4063624.0,
+      "reward": 0.7545166015625,
+      "reward_std": 0.008906117640435696,
+      "rewards//mean": 0.7545166015625,
+      "rewards//std": 0.027163511142134666,
+      "step": 470
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0942,
+      "grad_norm": 2.641265630722046,
+      "kl": 0.19954989198595285,
+      "learning_rate": 9.82341456229428e-07,
+      "loss": 0.008,
+      "num_tokens": 4072256.0,
+      "reward": 0.77545166015625,
+      "reward_std": 0.007894563488662243,
+      "rewards//mean": 0.77545166015625,
+      "rewards//std": 0.02620847150683403,
+      "step": 471
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0944,
+      "grad_norm": 2.660736083984375,
+      "kl": 0.19904836919158697,
+      "learning_rate": 9.822577692786272e-07,
+      "loss": 0.008,
+      "num_tokens": 4080920.0,
+      "reward": 0.75482177734375,
+      "reward_std": 0.008569098077714443,
+      "rewards//mean": 0.75482177734375,
+      "rewards//std": 0.022247202694416046,
+      "step": 472
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0946,
+      "grad_norm": 2.474003553390503,
+      "kl": 0.17544310074299574,
+      "learning_rate": 9.821738880744547e-07,
+      "loss": 0.007,
+      "num_tokens": 4089496.0,
+      "reward": 0.73590087890625,
+      "reward_std": 0.01006229966878891,
+      "rewards//mean": 0.73590087890625,
+      "rewards//std": 0.030430229380726814,
+      "step": 473
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0948,
+      "grad_norm": 2.616037130355835,
+      "kl": 0.19754528906196356,
+      "learning_rate": 9.820898126506979e-07,
+      "loss": 0.0079,
+      "num_tokens": 4098072.0,
+      "reward": 0.74658203125,
+      "reward_std": 0.00810934603214264,
+      "rewards//mean": 0.74658203125,
+      "rewards//std": 0.02306772582232952,
+      "step": 474
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.095,
+      "grad_norm": 2.8660085201263428,
+      "kl": 0.19248671364039183,
+      "learning_rate": 9.820055430412219e-07,
+      "loss": 0.0077,
+      "num_tokens": 4106672.0,
+      "reward": 0.70306396484375,
+      "reward_std": 0.011695638298988342,
+      "rewards//mean": 0.70306396484375,
+      "rewards//std": 0.040045253932476044,
+      "step": 475
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0952,
+      "grad_norm": 2.6622281074523926,
+      "kl": 0.19109869375824928,
+      "learning_rate": 9.81921079279971e-07,
+      "loss": 0.0076,
+      "num_tokens": 4115384.0,
+      "reward": 0.744873046875,
+      "reward_std": 0.009246986359357834,
+      "rewards//mean": 0.744873046875,
+      "rewards//std": 0.02397768571972847,
+      "step": 476
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0954,
+      "grad_norm": 2.4832065105438232,
+      "kl": 0.17988140787929296,
+      "learning_rate": 9.81836421400967e-07,
+      "loss": 0.0072,
+      "num_tokens": 4124064.0,
+      "reward": 0.73858642578125,
+      "reward_std": 0.007093518041074276,
+      "rewards//mean": 0.73858642578125,
+      "rewards//std": 0.03044365718960762,
+      "step": 477
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0956,
+      "grad_norm": 2.5871036052703857,
+      "kl": 0.18412148859351873,
+      "learning_rate": 9.817515694383102e-07,
+      "loss": 0.0074,
+      "num_tokens": 4132704.0,
+      "reward": 0.74658203125,
+      "reward_std": 0.010728240013122559,
+      "rewards//mean": 0.74658203125,
+      "rewards//std": 0.04038258269429207,
+      "step": 478
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0958,
+      "grad_norm": 2.662102460861206,
+      "kl": 0.19116327725350857,
+      "learning_rate": 9.816665234261786e-07,
+      "loss": 0.0076,
+      "num_tokens": 4141304.0,
+      "reward": 0.76910400390625,
+      "reward_std": 0.008907128125429153,
+      "rewards//mean": 0.76910400390625,
+      "rewards//std": 0.030226586386561394,
+      "step": 479
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.096,
+      "grad_norm": 2.553114175796509,
+      "kl": 0.19382564909756184,
+      "learning_rate": 9.81581283398829e-07,
+      "loss": 0.0078,
+      "num_tokens": 4149840.0,
+      "reward": 0.779541015625,
+      "reward_std": 0.010796980001032352,
+      "rewards//mean": 0.779541015625,
+      "rewards//std": 0.03200715780258179,
+      "step": 480
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0962,
+      "grad_norm": 2.7164504528045654,
+      "kl": 0.20583349466323853,
+      "learning_rate": 9.814958493905962e-07,
+      "loss": 0.0082,
+      "num_tokens": 4158448.0,
+      "reward": 0.7186279296875,
+      "reward_std": 0.007095596753060818,
+      "rewards//mean": 0.7186279296875,
+      "rewards//std": 0.037109989672899246,
+      "step": 481
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0964,
+      "grad_norm": 2.745314836502075,
+      "kl": 0.1839658385142684,
+      "learning_rate": 9.814102214358926e-07,
+      "loss": 0.0074,
+      "num_tokens": 4167072.0,
+      "reward": 0.738525390625,
+      "reward_std": 0.009960859082639217,
+      "rewards//mean": 0.738525390625,
+      "rewards//std": 0.03948724642395973,
+      "step": 482
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0966,
+      "grad_norm": 2.495893716812134,
+      "kl": 0.19855735450983047,
+      "learning_rate": 9.813243995692097e-07,
+      "loss": 0.0079,
+      "num_tokens": 4175656.0,
+      "reward": 0.7696533203125,
+      "reward_std": 0.010801002383232117,
+      "rewards//mean": 0.7696533203125,
+      "rewards//std": 0.028872955590486526,
+      "step": 483
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0968,
+      "grad_norm": 2.618044376373291,
+      "kl": 0.20143710542470217,
+      "learning_rate": 9.81238383825116e-07,
+      "loss": 0.0081,
+      "num_tokens": 4184232.0,
+      "reward": 0.741943359375,
+      "reward_std": 0.009049217216670513,
+      "rewards//mean": 0.741943359375,
+      "rewards//std": 0.025741152465343475,
+      "step": 484
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.097,
+      "grad_norm": 2.513794183731079,
+      "kl": 0.21006046421825886,
+      "learning_rate": 9.81152174238259e-07,
+      "loss": 0.0084,
+      "num_tokens": 4192904.0,
+      "reward": 0.72381591796875,
+      "reward_std": 0.00830012746155262,
+      "rewards//mean": 0.72381591796875,
+      "rewards//std": 0.032827552407979965,
+      "step": 485
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0972,
+      "grad_norm": 2.6051175594329834,
+      "kl": 0.18188222404569387,
+      "learning_rate": 9.810657708433635e-07,
+      "loss": 0.0073,
+      "num_tokens": 4201520.0,
+      "reward": 0.73992919921875,
+      "reward_std": 0.012143421918153763,
+      "rewards//mean": 0.73992919921875,
+      "rewards//std": 0.023102398961782455,
+      "step": 486
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0974,
+      "grad_norm": 2.344669818878174,
+      "kl": 0.20672739390283823,
+      "learning_rate": 9.809791736752332e-07,
+      "loss": 0.0083,
+      "num_tokens": 4210128.0,
+      "reward": 0.76385498046875,
+      "reward_std": 0.007372348103672266,
+      "rewards//mean": 0.76385498046875,
+      "rewards//std": 0.025873055681586266,
+      "step": 487
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0976,
+      "grad_norm": 2.6771304607391357,
+      "kl": 0.19675791915506124,
+      "learning_rate": 9.808923827687492e-07,
+      "loss": 0.0079,
+      "num_tokens": 4218696.0,
+      "reward": 0.732177734375,
+      "reward_std": 0.009250756353139877,
+      "rewards//mean": 0.732177734375,
+      "rewards//std": 0.032651446759700775,
+      "step": 488
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0978,
+      "grad_norm": 2.732707977294922,
+      "kl": 0.20094926748424768,
+      "learning_rate": 9.80805398158871e-07,
+      "loss": 0.008,
+      "num_tokens": 4227304.0,
+      "reward": 0.77362060546875,
+      "reward_std": 0.010892328806221485,
+      "rewards//mean": 0.77362060546875,
+      "rewards//std": 0.029110724106431007,
+      "step": 489
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.098,
+      "grad_norm": 2.597862958908081,
+      "kl": 0.2329936372116208,
+      "learning_rate": 9.80718219880636e-07,
+      "loss": 0.0093,
+      "num_tokens": 4235968.0,
+      "reward": 0.739990234375,
+      "reward_std": 0.007593792397528887,
+      "rewards//mean": 0.739990234375,
+      "rewards//std": 0.03062269650399685,
+      "step": 490
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0982,
+      "grad_norm": 2.4808294773101807,
+      "kl": 0.20427713077515364,
+      "learning_rate": 9.806308479691594e-07,
+      "loss": 0.0082,
+      "num_tokens": 4244616.0,
+      "reward": 0.74407958984375,
+      "reward_std": 0.008033126592636108,
+      "rewards//mean": 0.74407958984375,
+      "rewards//std": 0.028433604165911674,
+      "step": 491
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0984,
+      "grad_norm": 2.5725882053375244,
+      "kl": 0.2031407980248332,
+      "learning_rate": 9.805432824596347e-07,
+      "loss": 0.0081,
+      "num_tokens": 4253168.0,
+      "reward": 0.7191162109375,
+      "reward_std": 0.008553249761462212,
+      "rewards//mean": 0.7191162109375,
+      "rewards//std": 0.03302644565701485,
+      "step": 492
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0986,
+      "grad_norm": 2.347768783569336,
+      "kl": 0.19026768300682306,
+      "learning_rate": 9.804555233873332e-07,
+      "loss": 0.0076,
+      "num_tokens": 4261768.0,
+      "reward": 0.75909423828125,
+      "reward_std": 0.008124874904751778,
+      "rewards//mean": 0.75909423828125,
+      "rewards//std": 0.024676403030753136,
+      "step": 493
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0988,
+      "grad_norm": 2.4123456478118896,
+      "kl": 0.19286382384598255,
+      "learning_rate": 9.803675707876048e-07,
+      "loss": 0.0077,
+      "num_tokens": 4270328.0,
+      "reward": 0.72900390625,
+      "reward_std": 0.008946258574724197,
+      "rewards//mean": 0.72900390625,
+      "rewards//std": 0.03177647665143013,
+      "step": 494
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.099,
+      "grad_norm": 2.5037314891815186,
+      "kl": 0.20618513878434896,
+      "learning_rate": 9.80279424695876e-07,
+      "loss": 0.0082,
+      "num_tokens": 4279016.0,
+      "reward": 0.74676513671875,
+      "reward_std": 0.008192609064280987,
+      "rewards//mean": 0.74676513671875,
+      "rewards//std": 0.022867942228913307,
+      "step": 495
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0992,
+      "grad_norm": 2.2900962829589844,
+      "kl": 0.19684153143316507,
+      "learning_rate": 9.801910851476524e-07,
+      "loss": 0.0079,
+      "num_tokens": 4287632.0,
+      "reward": 0.7484130859375,
+      "reward_std": 0.008924616500735283,
+      "rewards//mean": 0.7484130859375,
+      "rewards//std": 0.032028671354055405,
+      "step": 496
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0994,
+      "grad_norm": 2.4447147846221924,
+      "kl": 0.2063806215301156,
+      "learning_rate": 9.80102552178517e-07,
+      "loss": 0.0083,
+      "num_tokens": 4296256.0,
+      "reward": 0.73052978515625,
+      "reward_std": 0.006515363696962595,
+      "rewards//mean": 0.73052978515625,
+      "rewards//std": 0.02408469282090664,
+      "step": 497
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0996,
+      "grad_norm": 2.3006718158721924,
+      "kl": 0.20592305064201355,
+      "learning_rate": 9.800138258241309e-07,
+      "loss": 0.0082,
+      "num_tokens": 4304944.0,
+      "reward": 0.76043701171875,
+      "reward_std": 0.007967970333993435,
+      "rewards//mean": 0.76043701171875,
+      "rewards//std": 0.038851458579301834,
+      "step": 498
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.0998,
+      "grad_norm": 2.6404166221618652,
+      "kl": 0.1996441464871168,
+      "learning_rate": 9.799249061202334e-07,
+      "loss": 0.008,
+      "num_tokens": 4313560.0,
+      "reward": 0.72857666015625,
+      "reward_std": 0.008022921159863472,
+      "rewards//mean": 0.72857666015625,
+      "rewards//std": 0.029675686731934547,
+      "step": 499
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1,
+      "grad_norm": 2.557433605194092,
+      "kl": 0.1909168316051364,
+      "learning_rate": 9.798357931026412e-07,
+      "loss": 0.0076,
+      "num_tokens": 4322208.0,
+      "reward": 0.76361083984375,
+      "reward_std": 0.007821734994649887,
+      "rewards//mean": 0.76361083984375,
+      "rewards//std": 0.025229064747691154,
+      "step": 500
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1002,
+      "grad_norm": 2.2387688159942627,
+      "kl": 0.19993739295750856,
+      "learning_rate": 9.797464868072486e-07,
+      "loss": 0.008,
+      "num_tokens": 4330888.0,
+      "reward": 0.75006103515625,
+      "reward_std": 0.007695702835917473,
+      "rewards//mean": 0.75006103515625,
+      "rewards//std": 0.028172602877020836,
+      "step": 501
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1004,
+      "grad_norm": 2.7008557319641113,
+      "kl": 0.19917786587029696,
+      "learning_rate": 9.796569872700287e-07,
+      "loss": 0.008,
+      "num_tokens": 4339480.0,
+      "reward": 0.7283935546875,
+      "reward_std": 0.011237498372793198,
+      "rewards//mean": 0.7283935546875,
+      "rewards//std": 0.039598070085048676,
+      "step": 502
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1006,
+      "grad_norm": 2.4576563835144043,
+      "kl": 0.2003437550738454,
+      "learning_rate": 9.795672945270316e-07,
+      "loss": 0.008,
+      "num_tokens": 4348208.0,
+      "reward": 0.71051025390625,
+      "reward_std": 0.007731921039521694,
+      "rewards//mean": 0.71051025390625,
+      "rewards//std": 0.03753923252224922,
+      "step": 503
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1008,
+      "grad_norm": 2.464399814605713,
+      "kl": 0.1987034259364009,
+      "learning_rate": 9.794774086143857e-07,
+      "loss": 0.0079,
+      "num_tokens": 4356904.0,
+      "reward": 0.75030517578125,
+      "reward_std": 0.008051925338804722,
+      "rewards//mean": 0.75030517578125,
+      "rewards//std": 0.025308741256594658,
+      "step": 504
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.101,
+      "grad_norm": 2.399820327758789,
+      "kl": 0.19125232845544815,
+      "learning_rate": 9.79387329568297e-07,
+      "loss": 0.0077,
+      "num_tokens": 4365528.0,
+      "reward": 0.75128173828125,
+      "reward_std": 0.010841385461390018,
+      "rewards//mean": 0.75128173828125,
+      "rewards//std": 0.0275955218821764,
+      "step": 505
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1012,
+      "grad_norm": 2.731841564178467,
+      "kl": 0.20694850012660027,
+      "learning_rate": 9.792970574250493e-07,
+      "loss": 0.0083,
+      "num_tokens": 4374120.0,
+      "reward": 0.73114013671875,
+      "reward_std": 0.009580669924616814,
+      "rewards//mean": 0.73114013671875,
+      "rewards//std": 0.020978784188628197,
+      "step": 506
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1014,
+      "grad_norm": 2.4787447452545166,
+      "kl": 0.18850557878613472,
+      "learning_rate": 9.79206592221004e-07,
+      "loss": 0.0075,
+      "num_tokens": 4382808.0,
+      "reward": 0.759033203125,
+      "reward_std": 0.010051162913441658,
+      "rewards//mean": 0.759033203125,
+      "rewards//std": 0.022892504930496216,
+      "step": 507
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1016,
+      "grad_norm": 2.734656572341919,
+      "kl": 0.19833427853882313,
+      "learning_rate": 9.791159339926008e-07,
+      "loss": 0.0079,
+      "num_tokens": 4391536.0,
+      "reward": 0.76953125,
+      "reward_std": 0.009782131761312485,
+      "rewards//mean": 0.76953125,
+      "rewards//std": 0.027134796604514122,
+      "step": 508
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1018,
+      "grad_norm": 2.514674425125122,
+      "kl": 0.2131390981376171,
+      "learning_rate": 9.790250827763565e-07,
+      "loss": 0.0085,
+      "num_tokens": 4400152.0,
+      "reward": 0.75579833984375,
+      "reward_std": 0.009274055249989033,
+      "rewards//mean": 0.75579833984375,
+      "rewards//std": 0.037097085267305374,
+      "step": 509
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.102,
+      "grad_norm": 2.802415609359741,
+      "kl": 0.1829777229577303,
+      "learning_rate": 9.789340386088662e-07,
+      "loss": 0.0073,
+      "num_tokens": 4408712.0,
+      "reward": 0.72381591796875,
+      "reward_std": 0.00830510351806879,
+      "rewards//mean": 0.72381591796875,
+      "rewards//std": 0.03307013213634491,
+      "step": 510
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1022,
+      "grad_norm": 2.1923043727874756,
+      "kl": 0.19948930945247412,
+      "learning_rate": 9.788428015268026e-07,
+      "loss": 0.008,
+      "num_tokens": 4417376.0,
+      "reward": 0.7576904296875,
+      "reward_std": 0.00736301438882947,
+      "rewards//mean": 0.7576904296875,
+      "rewards//std": 0.023488791659474373,
+      "step": 511
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1024,
+      "grad_norm": 2.5372860431671143,
+      "kl": 0.20013653114438057,
+      "learning_rate": 9.787513715669157e-07,
+      "loss": 0.008,
+      "num_tokens": 4426096.0,
+      "reward": 0.75469970703125,
+      "reward_std": 0.009306471794843674,
+      "rewards//mean": 0.75469970703125,
+      "rewards//std": 0.0311115775257349,
+      "step": 512
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1026,
+      "grad_norm": 2.5075161457061768,
+      "kl": 0.21732639521360397,
+      "learning_rate": 9.786597487660335e-07,
+      "loss": 0.0087,
+      "num_tokens": 4434696.0,
+      "reward": 0.74029541015625,
+      "reward_std": 0.011468115262687206,
+      "rewards//mean": 0.74029541015625,
+      "rewards//std": 0.036023881286382675,
+      "step": 513
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1028,
+      "grad_norm": 2.4096226692199707,
+      "kl": 0.2048845263198018,
+      "learning_rate": 9.78567933161062e-07,
+      "loss": 0.0082,
+      "num_tokens": 4443352.0,
+      "reward": 0.760986328125,
+      "reward_std": 0.007425335701555014,
+      "rewards//mean": 0.760986328125,
+      "rewards//std": 0.020633282139897346,
+      "step": 514
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.103,
+      "grad_norm": 2.4639785289764404,
+      "kl": 0.19642549473792315,
+      "learning_rate": 9.78475924788984e-07,
+      "loss": 0.0079,
+      "num_tokens": 4452008.0,
+      "reward": 0.7655029296875,
+      "reward_std": 0.00605900539085269,
+      "rewards//mean": 0.7655029296875,
+      "rewards//std": 0.026829317212104797,
+      "step": 515
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1032,
+      "grad_norm": 2.3441789150238037,
+      "kl": 0.19151584152132273,
+      "learning_rate": 9.783837236868609e-07,
+      "loss": 0.0077,
+      "num_tokens": 4460584.0,
+      "reward": 0.75494384765625,
+      "reward_std": 0.007256645243614912,
+      "rewards//mean": 0.75494384765625,
+      "rewards//std": 0.013319531455636024,
+      "step": 516
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1034,
+      "grad_norm": 2.703347682952881,
+      "kl": 0.1911890273913741,
+      "learning_rate": 9.782913298918308e-07,
+      "loss": 0.0076,
+      "num_tokens": 4469208.0,
+      "reward": 0.77349853515625,
+      "reward_std": 0.010297955945134163,
+      "rewards//mean": 0.77349853515625,
+      "rewards//std": 0.023994015529751778,
+      "step": 517
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1036,
+      "grad_norm": 2.3911008834838867,
+      "kl": 0.2087996369227767,
+      "learning_rate": 9.781987434411106e-07,
+      "loss": 0.0084,
+      "num_tokens": 4477880.0,
+      "reward": 0.7352294921875,
+      "reward_std": 0.006590959616005421,
+      "rewards//mean": 0.7352294921875,
+      "rewards//std": 0.02513747289776802,
+      "step": 518
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1038,
+      "grad_norm": 2.5501441955566406,
+      "kl": 0.19869405683130026,
+      "learning_rate": 9.781059643719936e-07,
+      "loss": 0.0079,
+      "num_tokens": 4486520.0,
+      "reward": 0.7608642578125,
+      "reward_std": 0.00740392180159688,
+      "rewards//mean": 0.7608642578125,
+      "rewards//std": 0.030540280044078827,
+      "step": 519
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.104,
+      "grad_norm": 2.442887306213379,
+      "kl": 0.2156991232186556,
+      "learning_rate": 9.780129927218511e-07,
+      "loss": 0.0086,
+      "num_tokens": 4495064.0,
+      "reward": 0.75372314453125,
+      "reward_std": 0.008730370551347733,
+      "rewards//mean": 0.75372314453125,
+      "rewards//std": 0.027449216693639755,
+      "step": 520
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1042,
+      "grad_norm": 2.282963275909424,
+      "kl": 0.19761809334158897,
+      "learning_rate": 9.779198285281326e-07,
+      "loss": 0.0079,
+      "num_tokens": 4503728.0,
+      "reward": 0.73199462890625,
+      "reward_std": 0.008112097159028053,
+      "rewards//mean": 0.73199462890625,
+      "rewards//std": 0.03141709789633751,
+      "step": 521
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1044,
+      "grad_norm": 2.3329858779907227,
+      "kl": 0.2080096947029233,
+      "learning_rate": 9.77826471828364e-07,
+      "loss": 0.0083,
+      "num_tokens": 4512344.0,
+      "reward": 0.76318359375,
+      "reward_std": 0.007059612777084112,
+      "rewards//mean": 0.76318359375,
+      "rewards//std": 0.025233939290046692,
+      "step": 522
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1046,
+      "grad_norm": 2.312187910079956,
+      "kl": 0.18228135351091623,
+      "learning_rate": 9.777329226601501e-07,
+      "loss": 0.0073,
+      "num_tokens": 4520896.0,
+      "reward": 0.7510986328125,
+      "reward_std": 0.008694940246641636,
+      "rewards//mean": 0.7510986328125,
+      "rewards//std": 0.020850403234362602,
+      "step": 523
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1048,
+      "grad_norm": 2.6797306537628174,
+      "kl": 0.1950625041499734,
+      "learning_rate": 9.776391810611718e-07,
+      "loss": 0.0078,
+      "num_tokens": 4529480.0,
+      "reward": 0.73638916015625,
+      "reward_std": 0.008760758675634861,
+      "rewards//mean": 0.73638916015625,
+      "rewards//std": 0.022107960656285286,
+      "step": 524
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.105,
+      "grad_norm": 2.5822930335998535,
+      "kl": 0.20127020124346018,
+      "learning_rate": 9.775452470691885e-07,
+      "loss": 0.0081,
+      "num_tokens": 4538064.0,
+      "reward": 0.75830078125,
+      "reward_std": 0.007956849411129951,
+      "rewards//mean": 0.75830078125,
+      "rewards//std": 0.020487504079937935,
+      "step": 525
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1052,
+      "grad_norm": 2.515317440032959,
+      "kl": 0.19474153500050306,
+      "learning_rate": 9.774511207220368e-07,
+      "loss": 0.0078,
+      "num_tokens": 4546688.0,
+      "reward": 0.778076171875,
+      "reward_std": 0.008999904617667198,
+      "rewards//mean": 0.778076171875,
+      "rewards//std": 0.024517083540558815,
+      "step": 526
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1054,
+      "grad_norm": 2.553143262863159,
+      "kl": 0.20256055518984795,
+      "learning_rate": 9.77356802057631e-07,
+      "loss": 0.0081,
+      "num_tokens": 4555432.0,
+      "reward": 0.78192138671875,
+      "reward_std": 0.008142570033669472,
+      "rewards//mean": 0.78192138671875,
+      "rewards//std": 0.024024909362196922,
+      "step": 527
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1056,
+      "grad_norm": 2.4554619789123535,
+      "kl": 0.1910470724105835,
+      "learning_rate": 9.77262291113962e-07,
+      "loss": 0.0076,
+      "num_tokens": 4564064.0,
+      "reward": 0.773681640625,
+      "reward_std": 0.007831105962395668,
+      "rewards//mean": 0.773681640625,
+      "rewards//std": 0.022998061031103134,
+      "step": 528
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1058,
+      "grad_norm": 2.4189770221710205,
+      "kl": 0.219999672845006,
+      "learning_rate": 9.771675879290996e-07,
+      "loss": 0.0088,
+      "num_tokens": 4572752.0,
+      "reward": 0.75238037109375,
+      "reward_std": 0.006505347788333893,
+      "rewards//mean": 0.75238037109375,
+      "rewards//std": 0.024196309968829155,
+      "step": 529
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.106,
+      "grad_norm": 2.3663268089294434,
+      "kl": 0.19590437784790993,
+      "learning_rate": 9.770726925411897e-07,
+      "loss": 0.0078,
+      "num_tokens": 4581432.0,
+      "reward": 0.78173828125,
+      "reward_std": 0.007341471035033464,
+      "rewards//mean": 0.78173828125,
+      "rewards//std": 0.02225545234978199,
+      "step": 530
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1062,
+      "grad_norm": 2.2811291217803955,
+      "kl": 0.20613388065248728,
+      "learning_rate": 9.769776049884563e-07,
+      "loss": 0.0082,
+      "num_tokens": 4590056.0,
+      "reward": 0.75042724609375,
+      "reward_std": 0.008027258329093456,
+      "rewards//mean": 0.75042724609375,
+      "rewards//std": 0.030585041269659996,
+      "step": 531
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1064,
+      "grad_norm": 2.760805130004883,
+      "kl": 0.20790709182620049,
+      "learning_rate": 9.768823253092008e-07,
+      "loss": 0.0083,
+      "num_tokens": 4598776.0,
+      "reward": 0.74578857421875,
+      "reward_std": 0.008729961700737476,
+      "rewards//mean": 0.74578857421875,
+      "rewards//std": 0.020163437351584435,
+      "step": 532
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1066,
+      "grad_norm": 2.5373082160949707,
+      "kl": 0.2122226096689701,
+      "learning_rate": 9.767868535418014e-07,
+      "loss": 0.0085,
+      "num_tokens": 4607464.0,
+      "reward": 0.769775390625,
+      "reward_std": 0.009555306285619736,
+      "rewards//mean": 0.769775390625,
+      "rewards//std": 0.03326871246099472,
+      "step": 533
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1068,
+      "grad_norm": 2.4272828102111816,
+      "kl": 0.2178900595754385,
+      "learning_rate": 9.766911897247146e-07,
+      "loss": 0.0087,
+      "num_tokens": 4616104.0,
+      "reward": 0.7227783203125,
+      "reward_std": 0.007462595589458942,
+      "rewards//mean": 0.7227783203125,
+      "rewards//std": 0.03189416229724884,
+      "step": 534
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.107,
+      "grad_norm": 2.4738922119140625,
+      "kl": 0.2237789435312152,
+      "learning_rate": 9.765953338964734e-07,
+      "loss": 0.009,
+      "num_tokens": 4624760.0,
+      "reward": 0.756591796875,
+      "reward_std": 0.0054986486211419106,
+      "rewards//mean": 0.756591796875,
+      "rewards//std": 0.02899773046374321,
+      "step": 535
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1072,
+      "grad_norm": 2.638998031616211,
+      "kl": 0.23875396233052015,
+      "learning_rate": 9.76499286095689e-07,
+      "loss": 0.0096,
+      "num_tokens": 4633392.0,
+      "reward": 0.750732421875,
+      "reward_std": 0.0086929090321064,
+      "rewards//mean": 0.750732421875,
+      "rewards//std": 0.027240585535764694,
+      "step": 536
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1074,
+      "grad_norm": 2.5916919708251953,
+      "kl": 0.2088939379900694,
+      "learning_rate": 9.764030463610488e-07,
+      "loss": 0.0084,
+      "num_tokens": 4642072.0,
+      "reward": 0.7579345703125,
+      "reward_std": 0.008034957572817802,
+      "rewards//mean": 0.7579345703125,
+      "rewards//std": 0.024626431986689568,
+      "step": 537
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1076,
+      "grad_norm": 2.7041096687316895,
+      "kl": 0.20330458413809538,
+      "learning_rate": 9.763066147313189e-07,
+      "loss": 0.0081,
+      "num_tokens": 4650720.0,
+      "reward": 0.75616455078125,
+      "reward_std": 0.007733976934105158,
+      "rewards//mean": 0.75616455078125,
+      "rewards//std": 0.024278128519654274,
+      "step": 538
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1078,
+      "grad_norm": 2.899296283721924,
+      "kl": 0.21454135701060295,
+      "learning_rate": 9.762099912453412e-07,
+      "loss": 0.0086,
+      "num_tokens": 4659312.0,
+      "reward": 0.7630615234375,
+      "reward_std": 0.011397436261177063,
+      "rewards//mean": 0.7630615234375,
+      "rewards//std": 0.03030742146074772,
+      "step": 539
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.108,
+      "grad_norm": 2.7333781719207764,
+      "kl": 0.20671376585960388,
+      "learning_rate": 9.76113175942036e-07,
+      "loss": 0.0083,
+      "num_tokens": 4667896.0,
+      "reward": 0.74896240234375,
+      "reward_std": 0.008332246914505959,
+      "rewards//mean": 0.74896240234375,
+      "rewards//std": 0.024805502966046333,
+      "step": 540
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1082,
+      "grad_norm": 2.7489192485809326,
+      "kl": 0.22079274151474237,
+      "learning_rate": 9.760161688604007e-07,
+      "loss": 0.0088,
+      "num_tokens": 4676488.0,
+      "reward": 0.75518798828125,
+      "reward_std": 0.008646560832858086,
+      "rewards//mean": 0.75518798828125,
+      "rewards//std": 0.022315820679068565,
+      "step": 541
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1084,
+      "grad_norm": 2.610943555831909,
+      "kl": 0.20998156070709229,
+      "learning_rate": 9.759189700395095e-07,
+      "loss": 0.0084,
+      "num_tokens": 4685128.0,
+      "reward": 0.755126953125,
+      "reward_std": 0.009915570728480816,
+      "rewards//mean": 0.755126953125,
+      "rewards//std": 0.03310084342956543,
+      "step": 542
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1086,
+      "grad_norm": 2.5846469402313232,
+      "kl": 0.1863661976531148,
+      "learning_rate": 9.758215795185138e-07,
+      "loss": 0.0075,
+      "num_tokens": 4693824.0,
+      "reward": 0.7552490234375,
+      "reward_std": 0.007276731543242931,
+      "rewards//mean": 0.7552490234375,
+      "rewards//std": 0.04481577128171921,
+      "step": 543
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1088,
+      "grad_norm": 2.8750290870666504,
+      "kl": 0.2036522412672639,
+      "learning_rate": 9.757239973366428e-07,
+      "loss": 0.0081,
+      "num_tokens": 4702472.0,
+      "reward": 0.75543212890625,
+      "reward_std": 0.007248344831168652,
+      "rewards//mean": 0.75543212890625,
+      "rewards//std": 0.03401559218764305,
+      "step": 544
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.109,
+      "grad_norm": 2.3670542240142822,
+      "kl": 0.21758887451142073,
+      "learning_rate": 9.756262235332028e-07,
+      "loss": 0.0087,
+      "num_tokens": 4711104.0,
+      "reward": 0.78314208984375,
+      "reward_std": 0.006803870666772127,
+      "rewards//mean": 0.78314208984375,
+      "rewards//std": 0.020751703530550003,
+      "step": 545
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1092,
+      "grad_norm": 2.6635184288024902,
+      "kl": 0.21388847939670086,
+      "learning_rate": 9.755282581475767e-07,
+      "loss": 0.0086,
+      "num_tokens": 4719688.0,
+      "reward": 0.7152099609375,
+      "reward_std": 0.009670565836131573,
+      "rewards//mean": 0.7152099609375,
+      "rewards//std": 0.037010323256254196,
+      "step": 546
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1094,
+      "grad_norm": 2.5421857833862305,
+      "kl": 0.19934120308607817,
+      "learning_rate": 9.754301012192253e-07,
+      "loss": 0.008,
+      "num_tokens": 4728272.0,
+      "reward": 0.75390625,
+      "reward_std": 0.00879538431763649,
+      "rewards//mean": 0.75390625,
+      "rewards//std": 0.022851470857858658,
+      "step": 547
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1096,
+      "grad_norm": 2.671503782272339,
+      "kl": 0.20683455374091864,
+      "learning_rate": 9.753317527876856e-07,
+      "loss": 0.0083,
+      "num_tokens": 4736888.0,
+      "reward": 0.7462158203125,
+      "reward_std": 0.007687820121645927,
+      "rewards//mean": 0.7462158203125,
+      "rewards//std": 0.028830982744693756,
+      "step": 548
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1098,
+      "grad_norm": 2.4200358390808105,
+      "kl": 0.20728019066154957,
+      "learning_rate": 9.75233212892573e-07,
+      "loss": 0.0083,
+      "num_tokens": 4745456.0,
+      "reward": 0.76397705078125,
+      "reward_std": 0.010222984477877617,
+      "rewards//mean": 0.76397705078125,
+      "rewards//std": 0.023579727858304977,
+      "step": 549
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.11,
+      "grad_norm": 2.7071242332458496,
+      "kl": 0.23281041253358126,
+      "learning_rate": 9.75134481573579e-07,
+      "loss": 0.0093,
+      "num_tokens": 4754000.0,
+      "reward": 0.7882080078125,
+      "reward_std": 0.00951644778251648,
+      "rewards//mean": 0.7882080078125,
+      "rewards//std": 0.021926190704107285,
+      "step": 550
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1102,
+      "grad_norm": 2.5958337783813477,
+      "kl": 0.19507870636880398,
+      "learning_rate": 9.750355588704727e-07,
+      "loss": 0.0078,
+      "num_tokens": 4762808.0,
+      "reward": 0.75701904296875,
+      "reward_std": 0.00885623786598444,
+      "rewards//mean": 0.75701904296875,
+      "rewards//std": 0.029848111793398857,
+      "step": 551
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1104,
+      "grad_norm": 2.472012996673584,
+      "kl": 0.20976228173822165,
+      "learning_rate": 9.749364448231e-07,
+      "loss": 0.0084,
+      "num_tokens": 4771488.0,
+      "reward": 0.73773193359375,
+      "reward_std": 0.008529557846486568,
+      "rewards//mean": 0.73773193359375,
+      "rewards//std": 0.02323894202709198,
+      "step": 552
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1106,
+      "grad_norm": 2.439462900161743,
+      "kl": 0.21856457088142633,
+      "learning_rate": 9.748371394713841e-07,
+      "loss": 0.0087,
+      "num_tokens": 4780144.0,
+      "reward": 0.74310302734375,
+      "reward_std": 0.007541933096945286,
+      "rewards//mean": 0.74310302734375,
+      "rewards//std": 0.025569967925548553,
+      "step": 553
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1108,
+      "grad_norm": 2.706909656524658,
+      "kl": 0.1947801075875759,
+      "learning_rate": 9.747376428553253e-07,
+      "loss": 0.0078,
+      "num_tokens": 4788744.0,
+      "reward": 0.7108154296875,
+      "reward_std": 0.010071827098727226,
+      "rewards//mean": 0.7108154296875,
+      "rewards//std": 0.036030445247888565,
+      "step": 554
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.111,
+      "grad_norm": 2.587865114212036,
+      "kl": 0.23300563916563988,
+      "learning_rate": 9.746379550150008e-07,
+      "loss": 0.0093,
+      "num_tokens": 4797400.0,
+      "reward": 0.7410888671875,
+      "reward_std": 0.008374853990972042,
+      "rewards//mean": 0.7410888671875,
+      "rewards//std": 0.02552945725619793,
+      "step": 555
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1112,
+      "grad_norm": 2.6079254150390625,
+      "kl": 0.21863503381609917,
+      "learning_rate": 9.745380759905647e-07,
+      "loss": 0.0087,
+      "num_tokens": 4806144.0,
+      "reward": 0.79052734375,
+      "reward_std": 0.007233437150716782,
+      "rewards//mean": 0.79052734375,
+      "rewards//std": 0.02630539983510971,
+      "step": 556
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1114,
+      "grad_norm": 3.000269889831543,
+      "kl": 0.20787948928773403,
+      "learning_rate": 9.744380058222482e-07,
+      "loss": 0.0083,
+      "num_tokens": 4814792.0,
+      "reward": 0.763916015625,
+      "reward_std": 0.011713992804288864,
+      "rewards//mean": 0.763916015625,
+      "rewards//std": 0.022955898195505142,
+      "step": 557
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1116,
+      "grad_norm": 2.738211154937744,
+      "kl": 0.20346645265817642,
+      "learning_rate": 9.743377445503597e-07,
+      "loss": 0.0081,
+      "num_tokens": 4823488.0,
+      "reward": 0.7716064453125,
+      "reward_std": 0.007158038206398487,
+      "rewards//mean": 0.7716064453125,
+      "rewards//std": 0.03104759007692337,
+      "step": 558
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1118,
+      "grad_norm": 2.745866298675537,
+      "kl": 0.20644012559205294,
+      "learning_rate": 9.742372922152845e-07,
+      "loss": 0.0083,
+      "num_tokens": 4832160.0,
+      "reward": 0.77496337890625,
+      "reward_std": 0.013732454739511013,
+      "rewards//mean": 0.77496337890625,
+      "rewards//std": 0.03590012714266777,
+      "step": 559
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.112,
+      "grad_norm": 2.579831838607788,
+      "kl": 0.20006072893738747,
+      "learning_rate": 9.74136648857485e-07,
+      "loss": 0.008,
+      "num_tokens": 4840776.0,
+      "reward": 0.7215576171875,
+      "reward_std": 0.00754170399159193,
+      "rewards//mean": 0.7215576171875,
+      "rewards//std": 0.03687099739909172,
+      "step": 560
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1122,
+      "grad_norm": 2.6222269535064697,
+      "kl": 0.21760489791631699,
+      "learning_rate": 9.740358145174997e-07,
+      "loss": 0.0087,
+      "num_tokens": 4849480.0,
+      "reward": 0.7720947265625,
+      "reward_std": 0.010821899399161339,
+      "rewards//mean": 0.7720947265625,
+      "rewards//std": 0.031313635408878326,
+      "step": 561
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1124,
+      "grad_norm": 2.5947396755218506,
+      "kl": 0.21706217341125011,
+      "learning_rate": 9.73934789235945e-07,
+      "loss": 0.0087,
+      "num_tokens": 4858184.0,
+      "reward": 0.7625732421875,
+      "reward_std": 0.008415354415774345,
+      "rewards//mean": 0.7625732421875,
+      "rewards//std": 0.03192072734236717,
+      "step": 562
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1126,
+      "grad_norm": 2.336047410964966,
+      "kl": 0.18568948656320572,
+      "learning_rate": 9.73833573053514e-07,
+      "loss": 0.0074,
+      "num_tokens": 4866856.0,
+      "reward": 0.77105712890625,
+      "reward_std": 0.00783691368997097,
+      "rewards//mean": 0.77105712890625,
+      "rewards//std": 0.022494863718748093,
+      "step": 563
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1128,
+      "grad_norm": 2.3330979347229004,
+      "kl": 0.20027238689363003,
+      "learning_rate": 9.737321660109766e-07,
+      "loss": 0.008,
+      "num_tokens": 4875376.0,
+      "reward": 0.73626708984375,
+      "reward_std": 0.008788051083683968,
+      "rewards//mean": 0.73626708984375,
+      "rewards//std": 0.03000037744641304,
+      "step": 564
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.113,
+      "grad_norm": 2.5522613525390625,
+      "kl": 0.2055955696851015,
+      "learning_rate": 9.73630568149179e-07,
+      "loss": 0.0082,
+      "num_tokens": 4884008.0,
+      "reward": 0.721435546875,
+      "reward_std": 0.007860198616981506,
+      "rewards//mean": 0.721435546875,
+      "rewards//std": 0.03621755167841911,
+      "step": 565
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1132,
+      "grad_norm": 2.5428450107574463,
+      "kl": 0.19873892422765493,
+      "learning_rate": 9.735287795090454e-07,
+      "loss": 0.0079,
+      "num_tokens": 4892672.0,
+      "reward": 0.77001953125,
+      "reward_std": 0.009354211390018463,
+      "rewards//mean": 0.77001953125,
+      "rewards//std": 0.025794023647904396,
+      "step": 566
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1134,
+      "grad_norm": 2.505075454711914,
+      "kl": 0.20103717595338821,
+      "learning_rate": 9.734268001315759e-07,
+      "loss": 0.008,
+      "num_tokens": 4901272.0,
+      "reward": 0.73175048828125,
+      "reward_std": 0.011893498711287975,
+      "rewards//mean": 0.73175048828125,
+      "rewards//std": 0.03574080392718315,
+      "step": 567
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1136,
+      "grad_norm": 2.3751235008239746,
+      "kl": 0.23137819208204746,
+      "learning_rate": 9.733246300578482e-07,
+      "loss": 0.0093,
+      "num_tokens": 4909840.0,
+      "reward": 0.76470947265625,
+      "reward_std": 0.008945251815021038,
+      "rewards//mean": 0.76470947265625,
+      "rewards//std": 0.023465821519494057,
+      "step": 568
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1138,
+      "grad_norm": 2.253753423690796,
+      "kl": 0.18717945273965597,
+      "learning_rate": 9.73222269329016e-07,
+      "loss": 0.0075,
+      "num_tokens": 4918384.0,
+      "reward": 0.75146484375,
+      "reward_std": 0.0072065722197294235,
+      "rewards//mean": 0.75146484375,
+      "rewards//std": 0.016374332830309868,
+      "step": 569
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.114,
+      "grad_norm": 2.406771183013916,
+      "kl": 0.20669802837073803,
+      "learning_rate": 9.731197179863103e-07,
+      "loss": 0.0083,
+      "num_tokens": 4927096.0,
+      "reward": 0.77728271484375,
+      "reward_std": 0.00788349099457264,
+      "rewards//mean": 0.77728271484375,
+      "rewards//std": 0.030165426433086395,
+      "step": 570
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1142,
+      "grad_norm": 2.6489546298980713,
+      "kl": 0.21308070421218872,
+      "learning_rate": 9.730169760710385e-07,
+      "loss": 0.0085,
+      "num_tokens": 4935776.0,
+      "reward": 0.74444580078125,
+      "reward_std": 0.007337694056332111,
+      "rewards//mean": 0.74444580078125,
+      "rewards//std": 0.025256050750613213,
+      "step": 571
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1144,
+      "grad_norm": 2.2121670246124268,
+      "kl": 0.20004741102457047,
+      "learning_rate": 9.729140436245856e-07,
+      "loss": 0.008,
+      "num_tokens": 4944400.0,
+      "reward": 0.73980712890625,
+      "reward_std": 0.007258987985551357,
+      "rewards//mean": 0.73980712890625,
+      "rewards//std": 0.02781897597014904,
+      "step": 572
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1146,
+      "grad_norm": 2.572361946105957,
+      "kl": 0.21485054399818182,
+      "learning_rate": 9.728109206884125e-07,
+      "loss": 0.0086,
+      "num_tokens": 4953000.0,
+      "reward": 0.75616455078125,
+      "reward_std": 0.00930515956133604,
+      "rewards//mean": 0.75616455078125,
+      "rewards//std": 0.02749660424888134,
+      "step": 573
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1148,
+      "grad_norm": 2.499732494354248,
+      "kl": 0.18966595456004143,
+      "learning_rate": 9.72707607304057e-07,
+      "loss": 0.0076,
+      "num_tokens": 4961696.0,
+      "reward": 0.76312255859375,
+      "reward_std": 0.006110053043812513,
+      "rewards//mean": 0.76312255859375,
+      "rewards//std": 0.026850959286093712,
+      "step": 574
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.115,
+      "grad_norm": 2.362006664276123,
+      "kl": 0.18402161914855242,
+      "learning_rate": 9.726041035131338e-07,
+      "loss": 0.0074,
+      "num_tokens": 4970296.0,
+      "reward": 0.741943359375,
+      "reward_std": 0.008008334785699844,
+      "rewards//mean": 0.741943359375,
+      "rewards//std": 0.024831196293234825,
+      "step": 575
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1152,
+      "grad_norm": 2.7022228240966797,
+      "kl": 0.24209712725132704,
+      "learning_rate": 9.72500409357334e-07,
+      "loss": 0.0097,
+      "num_tokens": 4979040.0,
+      "reward": 0.75054931640625,
+      "reward_std": 0.008825918659567833,
+      "rewards//mean": 0.75054931640625,
+      "rewards//std": 0.04422084242105484,
+      "step": 576
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1154,
+      "grad_norm": 2.5268378257751465,
+      "kl": 0.21274339873343706,
+      "learning_rate": 9.723965248784262e-07,
+      "loss": 0.0085,
+      "num_tokens": 4987720.0,
+      "reward": 0.79443359375,
+      "reward_std": 0.00982908345758915,
+      "rewards//mean": 0.79443359375,
+      "rewards//std": 0.018525702878832817,
+      "step": 577
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1156,
+      "grad_norm": 2.417515277862549,
+      "kl": 0.21194756217300892,
+      "learning_rate": 9.722924501182546e-07,
+      "loss": 0.0085,
+      "num_tokens": 4996352.0,
+      "reward": 0.7261962890625,
+      "reward_std": 0.0078034717589616776,
+      "rewards//mean": 0.7261962890625,
+      "rewards//std": 0.03793460503220558,
+      "step": 578
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1158,
+      "grad_norm": 2.5283761024475098,
+      "kl": 0.18611767049878836,
+      "learning_rate": 9.721881851187405e-07,
+      "loss": 0.0074,
+      "num_tokens": 5005168.0,
+      "reward": 0.75762939453125,
+      "reward_std": 0.010058899410068989,
+      "rewards//mean": 0.75762939453125,
+      "rewards//std": 0.03418472781777382,
+      "step": 579
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.116,
+      "grad_norm": 1.9342117309570312,
+      "kl": 0.1932758465409279,
+      "learning_rate": 9.720837299218818e-07,
+      "loss": 0.0077,
+      "num_tokens": 5013744.0,
+      "reward": 0.75384521484375,
+      "reward_std": 0.005680167116224766,
+      "rewards//mean": 0.75384521484375,
+      "rewards//std": 0.025079822167754173,
+      "step": 580
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1162,
+      "grad_norm": 2.3345181941986084,
+      "kl": 0.20730253774672747,
+      "learning_rate": 9.719790845697532e-07,
+      "loss": 0.0083,
+      "num_tokens": 5022304.0,
+      "reward": 0.7408447265625,
+      "reward_std": 0.009484879672527313,
+      "rewards//mean": 0.7408447265625,
+      "rewards//std": 0.031383171677589417,
+      "step": 581
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1164,
+      "grad_norm": 2.366091728210449,
+      "kl": 0.20875695161521435,
+      "learning_rate": 9.71874249104506e-07,
+      "loss": 0.0084,
+      "num_tokens": 5030944.0,
+      "reward": 0.70733642578125,
+      "reward_std": 0.00794148351997137,
+      "rewards//mean": 0.70733642578125,
+      "rewards//std": 0.029824772849678993,
+      "step": 582
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1166,
+      "grad_norm": 2.5741488933563232,
+      "kl": 0.19299913849681616,
+      "learning_rate": 9.717692235683674e-07,
+      "loss": 0.0077,
+      "num_tokens": 5039632.0,
+      "reward": 0.76080322265625,
+      "reward_std": 0.007838273420929909,
+      "rewards//mean": 0.76080322265625,
+      "rewards//std": 0.026335233822464943,
+      "step": 583
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1168,
+      "grad_norm": 2.336000919342041,
+      "kl": 0.20232780231162906,
+      "learning_rate": 9.716640080036423e-07,
+      "loss": 0.0081,
+      "num_tokens": 5048256.0,
+      "reward": 0.75787353515625,
+      "reward_std": 0.011394470930099487,
+      "rewards//mean": 0.75787353515625,
+      "rewards//std": 0.020225655287504196,
+      "step": 584
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.117,
+      "grad_norm": 2.3802225589752197,
+      "kl": 0.2070111045613885,
+      "learning_rate": 9.715586024527109e-07,
+      "loss": 0.0083,
+      "num_tokens": 5056808.0,
+      "reward": 0.757080078125,
+      "reward_std": 0.007744330447167158,
+      "rewards//mean": 0.757080078125,
+      "rewards//std": 0.0297480970621109,
+      "step": 585
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1172,
+      "grad_norm": 2.345366954803467,
+      "kl": 0.19000516179949045,
+      "learning_rate": 9.714530069580308e-07,
+      "loss": 0.0076,
+      "num_tokens": 5065400.0,
+      "reward": 0.7744140625,
+      "reward_std": 0.007273450493812561,
+      "rewards//mean": 0.7744140625,
+      "rewards//std": 0.030425317585468292,
+      "step": 586
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1174,
+      "grad_norm": 2.5098626613616943,
+      "kl": 0.21227385383099318,
+      "learning_rate": 9.71347221562136e-07,
+      "loss": 0.0085,
+      "num_tokens": 5074064.0,
+      "reward": 0.78564453125,
+      "reward_std": 0.006442628800868988,
+      "rewards//mean": 0.78564453125,
+      "rewards//std": 0.026406485587358475,
+      "step": 587
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1176,
+      "grad_norm": 2.6274147033691406,
+      "kl": 0.2008717842400074,
+      "learning_rate": 9.712412463076367e-07,
+      "loss": 0.008,
+      "num_tokens": 5082720.0,
+      "reward": 0.7396240234375,
+      "reward_std": 0.01041447278112173,
+      "rewards//mean": 0.7396240234375,
+      "rewards//std": 0.0307280533015728,
+      "step": 588
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1178,
+      "grad_norm": 2.2818710803985596,
+      "kl": 0.2134570386260748,
+      "learning_rate": 9.711350812372196e-07,
+      "loss": 0.0085,
+      "num_tokens": 5091344.0,
+      "reward": 0.73651123046875,
+      "reward_std": 0.00837304349988699,
+      "rewards//mean": 0.73651123046875,
+      "rewards//std": 0.02319786697626114,
+      "step": 589
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.118,
+      "grad_norm": 2.303163528442383,
+      "kl": 0.22205129265785217,
+      "learning_rate": 9.710287263936483e-07,
+      "loss": 0.0089,
+      "num_tokens": 5100024.0,
+      "reward": 0.76922607421875,
+      "reward_std": 0.009423606097698212,
+      "rewards//mean": 0.76922607421875,
+      "rewards//std": 0.01837875321507454,
+      "step": 590
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1182,
+      "grad_norm": 2.4563488960266113,
+      "kl": 0.21712797321379185,
+      "learning_rate": 9.709221818197623e-07,
+      "loss": 0.0087,
+      "num_tokens": 5108640.0,
+      "reward": 0.736572265625,
+      "reward_std": 0.008645838126540184,
+      "rewards//mean": 0.736572265625,
+      "rewards//std": 0.023050658404827118,
+      "step": 591
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1184,
+      "grad_norm": 2.8191616535186768,
+      "kl": 0.2220251578837633,
+      "learning_rate": 9.708154475584777e-07,
+      "loss": 0.0089,
+      "num_tokens": 5117224.0,
+      "reward": 0.737548828125,
+      "reward_std": 0.008272893726825714,
+      "rewards//mean": 0.737548828125,
+      "rewards//std": 0.029478194192051888,
+      "step": 592
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1186,
+      "grad_norm": 2.3526864051818848,
+      "kl": 0.22833502758294344,
+      "learning_rate": 9.707085236527873e-07,
+      "loss": 0.0091,
+      "num_tokens": 5125776.0,
+      "reward": 0.78204345703125,
+      "reward_std": 0.00702026579529047,
+      "rewards//mean": 0.78204345703125,
+      "rewards//std": 0.020448213443160057,
+      "step": 593
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1188,
+      "grad_norm": 2.4170947074890137,
+      "kl": 0.2121505094692111,
+      "learning_rate": 9.706014101457599e-07,
+      "loss": 0.0085,
+      "num_tokens": 5134408.0,
+      "reward": 0.776611328125,
+      "reward_std": 0.007152588106691837,
+      "rewards//mean": 0.776611328125,
+      "rewards//std": 0.020773665979504585,
+      "step": 594
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.119,
+      "grad_norm": 2.487192153930664,
+      "kl": 0.24224482756108046,
+      "learning_rate": 9.704941070805405e-07,
+      "loss": 0.0097,
+      "num_tokens": 5143040.0,
+      "reward": 0.7618408203125,
+      "reward_std": 0.007607296109199524,
+      "rewards//mean": 0.7618408203125,
+      "rewards//std": 0.020422007888555527,
+      "step": 595
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1192,
+      "grad_norm": 2.591724395751953,
+      "kl": 0.22526742238551378,
+      "learning_rate": 9.70386614500351e-07,
+      "loss": 0.009,
+      "num_tokens": 5151704.0,
+      "reward": 0.7568359375,
+      "reward_std": 0.007447593845427036,
+      "rewards//mean": 0.7568359375,
+      "rewards//std": 0.020587746053934097,
+      "step": 596
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1194,
+      "grad_norm": 2.794344186782837,
+      "kl": 0.2177692949771881,
+      "learning_rate": 9.702789324484896e-07,
+      "loss": 0.0087,
+      "num_tokens": 5160400.0,
+      "reward": 0.750732421875,
+      "reward_std": 0.010969633236527443,
+      "rewards//mean": 0.750732421875,
+      "rewards//std": 0.021952755749225616,
+      "step": 597
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1196,
+      "grad_norm": 2.484254837036133,
+      "kl": 0.22438115626573563,
+      "learning_rate": 9.701710609683305e-07,
+      "loss": 0.009,
+      "num_tokens": 5169024.0,
+      "reward": 0.7562255859375,
+      "reward_std": 0.007083019241690636,
+      "rewards//mean": 0.7562255859375,
+      "rewards//std": 0.026011131703853607,
+      "step": 598
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1198,
+      "grad_norm": 2.862764358520508,
+      "kl": 0.21322147641330957,
+      "learning_rate": 9.700630001033243e-07,
+      "loss": 0.0085,
+      "num_tokens": 5177672.0,
+      "reward": 0.759765625,
+      "reward_std": 0.006926173344254494,
+      "rewards//mean": 0.759765625,
+      "rewards//std": 0.025132955983281136,
+      "step": 599
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.12,
+      "grad_norm": 2.6096386909484863,
+      "kl": 0.22782303858548403,
+      "learning_rate": 9.699547498969978e-07,
+      "loss": 0.0091,
+      "num_tokens": 5186400.0,
+      "reward": 0.7332763671875,
+      "reward_std": 0.007928953506052494,
+      "rewards//mean": 0.7332763671875,
+      "rewards//std": 0.03339655324816704,
+      "step": 600
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1202,
+      "grad_norm": 2.481480121612549,
+      "kl": 0.2197017315775156,
+      "learning_rate": 9.698463103929541e-07,
+      "loss": 0.0088,
+      "num_tokens": 5195048.0,
+      "reward": 0.73583984375,
+      "reward_std": 0.00684010935947299,
+      "rewards//mean": 0.73583984375,
+      "rewards//std": 0.020237719640135765,
+      "step": 601
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1204,
+      "grad_norm": 2.598886013031006,
+      "kl": 0.248580701649189,
+      "learning_rate": 9.69737681634873e-07,
+      "loss": 0.0099,
+      "num_tokens": 5203712.0,
+      "reward": 0.7767333984375,
+      "reward_std": 0.008067253045737743,
+      "rewards//mean": 0.7767333984375,
+      "rewards//std": 0.0184171162545681,
+      "step": 602
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1206,
+      "grad_norm": 2.55953311920166,
+      "kl": 0.236043612472713,
+      "learning_rate": 9.696288636665097e-07,
+      "loss": 0.0094,
+      "num_tokens": 5212352.0,
+      "reward": 0.766845703125,
+      "reward_std": 0.007209985051304102,
+      "rewards//mean": 0.766845703125,
+      "rewards//std": 0.02043275348842144,
+      "step": 603
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1208,
+      "grad_norm": 2.678175210952759,
+      "kl": 0.231639688834548,
+      "learning_rate": 9.695198565316964e-07,
+      "loss": 0.0093,
+      "num_tokens": 5220912.0,
+      "reward": 0.76904296875,
+      "reward_std": 0.009503869339823723,
+      "rewards//mean": 0.76904296875,
+      "rewards//std": 0.03473345935344696,
+      "step": 604
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.121,
+      "grad_norm": 2.3178458213806152,
+      "kl": 0.2136409254744649,
+      "learning_rate": 9.69410660274341e-07,
+      "loss": 0.0085,
+      "num_tokens": 5229616.0,
+      "reward": 0.75732421875,
+      "reward_std": 0.0072447145357728004,
+      "rewards//mean": 0.75732421875,
+      "rewards//std": 0.020675791427493095,
+      "step": 605
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1212,
+      "grad_norm": 2.709106922149658,
+      "kl": 0.24635965004563332,
+      "learning_rate": 9.693012749384277e-07,
+      "loss": 0.0099,
+      "num_tokens": 5238192.0,
+      "reward": 0.73809814453125,
+      "reward_std": 0.01116020604968071,
+      "rewards//mean": 0.73809814453125,
+      "rewards//std": 0.028742311522364616,
+      "step": 606
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1214,
+      "grad_norm": 2.685481071472168,
+      "kl": 0.23443083930760622,
+      "learning_rate": 9.691917005680173e-07,
+      "loss": 0.0094,
+      "num_tokens": 5246720.0,
+      "reward": 0.7750244140625,
+      "reward_std": 0.00865564402192831,
+      "rewards//mean": 0.7750244140625,
+      "rewards//std": 0.029171306639909744,
+      "step": 607
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1216,
+      "grad_norm": 2.558425188064575,
+      "kl": 0.22762812487781048,
+      "learning_rate": 9.690819372072456e-07,
+      "loss": 0.0091,
+      "num_tokens": 5255328.0,
+      "reward": 0.74798583984375,
+      "reward_std": 0.009355241432785988,
+      "rewards//mean": 0.74798583984375,
+      "rewards//std": 0.024773750454187393,
+      "step": 608
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1218,
+      "grad_norm": 2.848205089569092,
+      "kl": 0.2626179587095976,
+      "learning_rate": 9.68971984900326e-07,
+      "loss": 0.0105,
+      "num_tokens": 5263992.0,
+      "reward": 0.7454833984375,
+      "reward_std": 0.007315634749829769,
+      "rewards//mean": 0.7454833984375,
+      "rewards//std": 0.02932450734078884,
+      "step": 609
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.122,
+      "grad_norm": 2.914008855819702,
+      "kl": 0.21053088549524546,
+      "learning_rate": 9.688618436915468e-07,
+      "loss": 0.0084,
+      "num_tokens": 5272632.0,
+      "reward": 0.745849609375,
+      "reward_std": 0.008161933161318302,
+      "rewards//mean": 0.745849609375,
+      "rewards//std": 0.025064200162887573,
+      "step": 610
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1222,
+      "grad_norm": 2.6426734924316406,
+      "kl": 0.26074648182839155,
+      "learning_rate": 9.68751513625273e-07,
+      "loss": 0.0104,
+      "num_tokens": 5281264.0,
+      "reward": 0.76171875,
+      "reward_std": 0.009589259512722492,
+      "rewards//mean": 0.76171875,
+      "rewards//std": 0.024793371558189392,
+      "step": 611
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1224,
+      "grad_norm": 3.003979206085205,
+      "kl": 0.23192260041832924,
+      "learning_rate": 9.686409947459457e-07,
+      "loss": 0.0093,
+      "num_tokens": 5290096.0,
+      "reward": 0.7344970703125,
+      "reward_std": 0.008901823312044144,
+      "rewards//mean": 0.7344970703125,
+      "rewards//std": 0.029090242460370064,
+      "step": 612
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1226,
+      "grad_norm": 2.669278860092163,
+      "kl": 0.2331440532580018,
+      "learning_rate": 9.685302870980817e-07,
+      "loss": 0.0093,
+      "num_tokens": 5298720.0,
+      "reward": 0.7698974609375,
+      "reward_std": 0.008679483085870743,
+      "rewards//mean": 0.7698974609375,
+      "rewards//std": 0.025303132832050323,
+      "step": 613
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1228,
+      "grad_norm": 3.0592589378356934,
+      "kl": 0.24727703537791967,
+      "learning_rate": 9.684193907262742e-07,
+      "loss": 0.0099,
+      "num_tokens": 5307344.0,
+      "reward": 0.743408203125,
+      "reward_std": 0.011624716222286224,
+      "rewards//mean": 0.743408203125,
+      "rewards//std": 0.03264402970671654,
+      "step": 614
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.123,
+      "grad_norm": 2.7208240032196045,
+      "kl": 0.24731595907360315,
+      "learning_rate": 9.68308305675192e-07,
+      "loss": 0.0099,
+      "num_tokens": 5315880.0,
+      "reward": 0.7630615234375,
+      "reward_std": 0.00930720753967762,
+      "rewards//mean": 0.7630615234375,
+      "rewards//std": 0.025922521948814392,
+      "step": 615
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1232,
+      "grad_norm": 2.5424563884735107,
+      "kl": 0.2442295728251338,
+      "learning_rate": 9.681970319895802e-07,
+      "loss": 0.0098,
+      "num_tokens": 5324592.0,
+      "reward": 0.76751708984375,
+      "reward_std": 0.009459644556045532,
+      "rewards//mean": 0.76751708984375,
+      "rewards//std": 0.02340639755129814,
+      "step": 616
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1234,
+      "grad_norm": 2.7117526531219482,
+      "kl": 0.23817005194723606,
+      "learning_rate": 9.6808556971426e-07,
+      "loss": 0.0095,
+      "num_tokens": 5333288.0,
+      "reward": 0.75787353515625,
+      "reward_std": 0.007240240462124348,
+      "rewards//mean": 0.75787353515625,
+      "rewards//std": 0.020943395793437958,
+      "step": 617
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1236,
+      "grad_norm": 2.368464231491089,
+      "kl": 0.23345900233834982,
+      "learning_rate": 9.679739188941283e-07,
+      "loss": 0.0093,
+      "num_tokens": 5341944.0,
+      "reward": 0.75018310546875,
+      "reward_std": 0.008537301793694496,
+      "rewards//mean": 0.75018310546875,
+      "rewards//std": 0.02675211802124977,
+      "step": 618
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1238,
+      "grad_norm": 2.8834733963012695,
+      "kl": 0.2347724586725235,
+      "learning_rate": 9.678620795741582e-07,
+      "loss": 0.0094,
+      "num_tokens": 5350712.0,
+      "reward": 0.8011474609375,
+      "reward_std": 0.0081472247838974,
+      "rewards//mean": 0.8011474609375,
+      "rewards//std": 0.01927182264626026,
+      "step": 619
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.124,
+      "grad_norm": 2.6222214698791504,
+      "kl": 0.23652953561395407,
+      "learning_rate": 9.677500517993982e-07,
+      "loss": 0.0095,
+      "num_tokens": 5359400.0,
+      "reward": 0.75927734375,
+      "reward_std": 0.0071931458078324795,
+      "rewards//mean": 0.75927734375,
+      "rewards//std": 0.02862468920648098,
+      "step": 620
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1242,
+      "grad_norm": 2.447474718093872,
+      "kl": 0.2144069541245699,
+      "learning_rate": 9.676378356149732e-07,
+      "loss": 0.0086,
+      "num_tokens": 5368040.0,
+      "reward": 0.74359130859375,
+      "reward_std": 0.005447229836136103,
+      "rewards//mean": 0.74359130859375,
+      "rewards//std": 0.035368263721466064,
+      "step": 621
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1244,
+      "grad_norm": 3.263336181640625,
+      "kl": 0.24912102241069078,
+      "learning_rate": 9.675254310660841e-07,
+      "loss": 0.01,
+      "num_tokens": 5376656.0,
+      "reward": 0.73675537109375,
+      "reward_std": 0.009956108406186104,
+      "rewards//mean": 0.73675537109375,
+      "rewards//std": 0.03202790021896362,
+      "step": 622
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1246,
+      "grad_norm": 2.958742618560791,
+      "kl": 0.2529023764654994,
+      "learning_rate": 9.674128381980071e-07,
+      "loss": 0.0101,
+      "num_tokens": 5385312.0,
+      "reward": 0.74224853515625,
+      "reward_std": 0.006682121194899082,
+      "rewards//mean": 0.74224853515625,
+      "rewards//std": 0.024054504930973053,
+      "step": 623
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1248,
+      "grad_norm": 2.909976005554199,
+      "kl": 0.2629548078402877,
+      "learning_rate": 9.67300057056095e-07,
+      "loss": 0.0105,
+      "num_tokens": 5393888.0,
+      "reward": 0.75115966796875,
+      "reward_std": 0.007848279550671577,
+      "rewards//mean": 0.75115966796875,
+      "rewards//std": 0.025100937113165855,
+      "step": 624
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.125,
+      "grad_norm": 2.7330803871154785,
+      "kl": 0.24556253291666508,
+      "learning_rate": 9.671870876857758e-07,
+      "loss": 0.0098,
+      "num_tokens": 5402496.0,
+      "reward": 0.76007080078125,
+      "reward_std": 0.008249986916780472,
+      "rewards//mean": 0.76007080078125,
+      "rewards//std": 0.02755819447338581,
+      "step": 625
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1252,
+      "grad_norm": 2.7917697429656982,
+      "kl": 0.266143218614161,
+      "learning_rate": 9.670739301325534e-07,
+      "loss": 0.0106,
+      "num_tokens": 5411160.0,
+      "reward": 0.76275634765625,
+      "reward_std": 0.007092661689966917,
+      "rewards//mean": 0.76275634765625,
+      "rewards//std": 0.04186524078249931,
+      "step": 626
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1254,
+      "grad_norm": 2.990387439727783,
+      "kl": 0.2544079925864935,
+      "learning_rate": 9.669605844420078e-07,
+      "loss": 0.0102,
+      "num_tokens": 5419800.0,
+      "reward": 0.78546142578125,
+      "reward_std": 0.007669455837458372,
+      "rewards//mean": 0.78546142578125,
+      "rewards//std": 0.026569290086627007,
+      "step": 627
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1256,
+      "grad_norm": 3.2159271240234375,
+      "kl": 0.28000779543071985,
+      "learning_rate": 9.668470506597946e-07,
+      "loss": 0.0112,
+      "num_tokens": 5428536.0,
+      "reward": 0.73834228515625,
+      "reward_std": 0.01008367445319891,
+      "rewards//mean": 0.73834228515625,
+      "rewards//std": 0.021637342870235443,
+      "step": 628
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1258,
+      "grad_norm": 3.7808144092559814,
+      "kl": 0.28342091385275126,
+      "learning_rate": 9.667333288316453e-07,
+      "loss": 0.0113,
+      "num_tokens": 5437144.0,
+      "reward": 0.78680419921875,
+      "reward_std": 0.007455417886376381,
+      "rewards//mean": 0.78680419921875,
+      "rewards//std": 0.02565566450357437,
+      "step": 629
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.126,
+      "grad_norm": 3.1781973838806152,
+      "kl": 0.34094157721847296,
+      "learning_rate": 9.66619419003367e-07,
+      "loss": 0.0136,
+      "num_tokens": 5445856.0,
+      "reward": 0.75860595703125,
+      "reward_std": 0.010273408144712448,
+      "rewards//mean": 0.75860595703125,
+      "rewards//std": 0.02941489778459072,
+      "step": 630
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1262,
+      "grad_norm": 4.200908660888672,
+      "kl": 0.31012608483433723,
+      "learning_rate": 9.665053212208426e-07,
+      "loss": 0.0124,
+      "num_tokens": 5454512.0,
+      "reward": 0.72613525390625,
+      "reward_std": 0.01104994397610426,
+      "rewards//mean": 0.72613525390625,
+      "rewards//std": 0.03484218940138817,
+      "step": 631
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1264,
+      "grad_norm": 3.3702046871185303,
+      "kl": 0.29447157122194767,
+      "learning_rate": 9.663910355300304e-07,
+      "loss": 0.0118,
+      "num_tokens": 5463144.0,
+      "reward": 0.7642822265625,
+      "reward_std": 0.00872592069208622,
+      "rewards//mean": 0.7642822265625,
+      "rewards//std": 0.02257656119763851,
+      "step": 632
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1266,
+      "grad_norm": 3.8665194511413574,
+      "kl": 0.2957277214154601,
+      "learning_rate": 9.66276561976965e-07,
+      "loss": 0.0118,
+      "num_tokens": 5471768.0,
+      "reward": 0.750244140625,
+      "reward_std": 0.00933067500591278,
+      "rewards//mean": 0.750244140625,
+      "rewards//std": 0.019610146060585976,
+      "step": 633
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1268,
+      "grad_norm": 3.341804027557373,
+      "kl": 0.2521153260022402,
+      "learning_rate": 9.661619006077561e-07,
+      "loss": 0.0101,
+      "num_tokens": 5480384.0,
+      "reward": 0.75299072265625,
+      "reward_std": 0.007986579090356827,
+      "rewards//mean": 0.75299072265625,
+      "rewards//std": 0.021505415439605713,
+      "step": 634
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.127,
+      "grad_norm": 3.5835232734680176,
+      "kl": 0.33276358991861343,
+      "learning_rate": 9.660470514685895e-07,
+      "loss": 0.0133,
+      "num_tokens": 5488984.0,
+      "reward": 0.73980712890625,
+      "reward_std": 0.008749248459935188,
+      "rewards//mean": 0.73980712890625,
+      "rewards//std": 0.024534286931157112,
+      "step": 635
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1272,
+      "grad_norm": 3.281963348388672,
+      "kl": 0.26525493059307337,
+      "learning_rate": 9.659320146057262e-07,
+      "loss": 0.0106,
+      "num_tokens": 5497624.0,
+      "reward": 0.81622314453125,
+      "reward_std": 0.008145196363329887,
+      "rewards//mean": 0.81622314453125,
+      "rewards//std": 0.019604645669460297,
+      "step": 636
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1274,
+      "grad_norm": 3.1588807106018066,
+      "kl": 0.3334341961890459,
+      "learning_rate": 9.65816790065503e-07,
+      "loss": 0.0133,
+      "num_tokens": 5506312.0,
+      "reward": 0.74041748046875,
+      "reward_std": 0.008933894336223602,
+      "rewards//mean": 0.74041748046875,
+      "rewards//std": 0.025957170873880386,
+      "step": 637
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1276,
+      "grad_norm": 4.310033798217773,
+      "kl": 0.3234546445310116,
+      "learning_rate": 9.657013778943327e-07,
+      "loss": 0.0129,
+      "num_tokens": 5514912.0,
+      "reward": 0.73516845703125,
+      "reward_std": 0.010786399245262146,
+      "rewards//mean": 0.73516845703125,
+      "rewards//std": 0.019138913601636887,
+      "step": 638
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1278,
+      "grad_norm": 3.3058111667633057,
+      "kl": 0.26327971275895834,
+      "learning_rate": 9.65585778138703e-07,
+      "loss": 0.0105,
+      "num_tokens": 5523536.0,
+      "reward": 0.78564453125,
+      "reward_std": 0.006881622597575188,
+      "rewards//mean": 0.78564453125,
+      "rewards//std": 0.02183798886835575,
+      "step": 639
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.128,
+      "grad_norm": 5.251916885375977,
+      "kl": 0.3826856277883053,
+      "learning_rate": 9.654699908451776e-07,
+      "loss": 0.0153,
+      "num_tokens": 5532184.0,
+      "reward": 0.730224609375,
+      "reward_std": 0.011996911838650703,
+      "rewards//mean": 0.730224609375,
+      "rewards//std": 0.02739132009446621,
+      "step": 640
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1282,
+      "grad_norm": 6.175998687744141,
+      "kl": 0.5581840332597494,
+      "learning_rate": 9.653540160603955e-07,
+      "loss": 0.0223,
+      "num_tokens": 5540808.0,
+      "reward": 0.777587890625,
+      "reward_std": 0.017355050891637802,
+      "rewards//mean": 0.777587890625,
+      "rewards//std": 0.033652350306510925,
+      "step": 641
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1284,
+      "grad_norm": 3.777766466140747,
+      "kl": 0.45173448603600264,
+      "learning_rate": 9.652378538310713e-07,
+      "loss": 0.0181,
+      "num_tokens": 5549448.0,
+      "reward": 0.72308349609375,
+      "reward_std": 0.010328175500035286,
+      "rewards//mean": 0.72308349609375,
+      "rewards//std": 0.030106153339147568,
+      "step": 642
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1286,
+      "grad_norm": 4.543262481689453,
+      "kl": 0.37777620926499367,
+      "learning_rate": 9.651215042039953e-07,
+      "loss": 0.0151,
+      "num_tokens": 5558080.0,
+      "reward": 0.784912109375,
+      "reward_std": 0.00804133340716362,
+      "rewards//mean": 0.784912109375,
+      "rewards//std": 0.024118687957525253,
+      "step": 643
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1288,
+      "grad_norm": 5.618739128112793,
+      "kl": 0.41111266426742077,
+      "learning_rate": 9.650049672260333e-07,
+      "loss": 0.0164,
+      "num_tokens": 5566712.0,
+      "reward": 0.777587890625,
+      "reward_std": 0.009358042851090431,
+      "rewards//mean": 0.777587890625,
+      "rewards//std": 0.025352440774440765,
+      "step": 644
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.129,
+      "grad_norm": 4.500043869018555,
+      "kl": 0.4685266828164458,
+      "learning_rate": 9.648882429441256e-07,
+      "loss": 0.0187,
+      "num_tokens": 5575304.0,
+      "reward": 0.76861572265625,
+      "reward_std": 0.012531119398772717,
+      "rewards//mean": 0.76861572265625,
+      "rewards//std": 0.028879445046186447,
+      "step": 645
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1292,
+      "grad_norm": 4.578500747680664,
+      "kl": 0.4764310959726572,
+      "learning_rate": 9.647713314052895e-07,
+      "loss": 0.0191,
+      "num_tokens": 5584080.0,
+      "reward": 0.7603759765625,
+      "reward_std": 0.010578645393252373,
+      "rewards//mean": 0.7603759765625,
+      "rewards//std": 0.030641227960586548,
+      "step": 646
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1294,
+      "grad_norm": 5.9160237312316895,
+      "kl": 0.40606721490621567,
+      "learning_rate": 9.646542326566168e-07,
+      "loss": 0.0162,
+      "num_tokens": 5592720.0,
+      "reward": 0.76458740234375,
+      "reward_std": 0.0073903873562812805,
+      "rewards//mean": 0.76458740234375,
+      "rewards//std": 0.01999380625784397,
+      "step": 647
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1296,
+      "grad_norm": 4.43061637878418,
+      "kl": 0.4973544552922249,
+      "learning_rate": 9.645369467452745e-07,
+      "loss": 0.0199,
+      "num_tokens": 5601432.0,
+      "reward": 0.7669677734375,
+      "reward_std": 0.008010782301425934,
+      "rewards//mean": 0.7669677734375,
+      "rewards//std": 0.012856454588472843,
+      "step": 648
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1298,
+      "grad_norm": 5.037925720214844,
+      "kl": 0.5347670819610357,
+      "learning_rate": 9.644194737185057e-07,
+      "loss": 0.0214,
+      "num_tokens": 5610040.0,
+      "reward": 0.7193603515625,
+      "reward_std": 0.009534362703561783,
+      "rewards//mean": 0.7193603515625,
+      "rewards//std": 0.025828920304775238,
+      "step": 649
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.13,
+      "grad_norm": 5.004450798034668,
+      "kl": 0.516712237149477,
+      "learning_rate": 9.643018136236286e-07,
+      "loss": 0.0207,
+      "num_tokens": 5618752.0,
+      "reward": 0.73358154296875,
+      "reward_std": 0.007627109065651894,
+      "rewards//mean": 0.73358154296875,
+      "rewards//std": 0.02682219259440899,
+      "step": 650
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1302,
+      "grad_norm": 5.551060676574707,
+      "kl": 0.47126047126948833,
+      "learning_rate": 9.641839665080363e-07,
+      "loss": 0.0189,
+      "num_tokens": 5627320.0,
+      "reward": 0.759521484375,
+      "reward_std": 0.011706827208399773,
+      "rewards//mean": 0.759521484375,
+      "rewards//std": 0.01942399889230728,
+      "step": 651
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1304,
+      "grad_norm": 6.075092792510986,
+      "kl": 0.5999239925295115,
+      "learning_rate": 9.640659324191978e-07,
+      "loss": 0.024,
+      "num_tokens": 5635952.0,
+      "reward": 0.77130126953125,
+      "reward_std": 0.010738062672317028,
+      "rewards//mean": 0.77130126953125,
+      "rewards//std": 0.029699651524424553,
+      "step": 652
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1306,
+      "grad_norm": 5.55996036529541,
+      "kl": 0.5250953948125243,
+      "learning_rate": 9.639477114046572e-07,
+      "loss": 0.021,
+      "num_tokens": 5644616.0,
+      "reward": 0.75323486328125,
+      "reward_std": 0.009308221749961376,
+      "rewards//mean": 0.75323486328125,
+      "rewards//std": 0.021435614675283432,
+      "step": 653
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1308,
+      "grad_norm": 4.967021465301514,
+      "kl": 0.4518276769667864,
+      "learning_rate": 9.63829303512034e-07,
+      "loss": 0.0181,
+      "num_tokens": 5653272.0,
+      "reward": 0.7772216796875,
+      "reward_std": 0.008542926982045174,
+      "rewards//mean": 0.7772216796875,
+      "rewards//std": 0.022433962672948837,
+      "step": 654
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.131,
+      "grad_norm": 6.460659027099609,
+      "kl": 0.5484109930694103,
+      "learning_rate": 9.63710708789023e-07,
+      "loss": 0.0219,
+      "num_tokens": 5661888.0,
+      "reward": 0.7843017578125,
+      "reward_std": 0.009501149877905846,
+      "rewards//mean": 0.7843017578125,
+      "rewards//std": 0.018021628260612488,
+      "step": 655
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1312,
+      "grad_norm": 7.013893127441406,
+      "kl": 0.7591905985027552,
+      "learning_rate": 9.635919272833937e-07,
+      "loss": 0.0304,
+      "num_tokens": 5670504.0,
+      "reward": 0.7396240234375,
+      "reward_std": 0.012006670236587524,
+      "rewards//mean": 0.7396240234375,
+      "rewards//std": 0.029880868270993233,
+      "step": 656
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1314,
+      "grad_norm": 6.466878414154053,
+      "kl": 0.701962748542428,
+      "learning_rate": 9.634729590429916e-07,
+      "loss": 0.0281,
+      "num_tokens": 5679168.0,
+      "reward": 0.763427734375,
+      "reward_std": 0.011417343281209469,
+      "rewards//mean": 0.763427734375,
+      "rewards//std": 0.029780646786093712,
+      "step": 657
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1316,
+      "grad_norm": 5.745469570159912,
+      "kl": 0.6625272277742624,
+      "learning_rate": 9.63353804115737e-07,
+      "loss": 0.0265,
+      "num_tokens": 5687896.0,
+      "reward": 0.769287109375,
+      "reward_std": 0.007695325184613466,
+      "rewards//mean": 0.769287109375,
+      "rewards//std": 0.02231387048959732,
+      "step": 658
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1318,
+      "grad_norm": 7.092409133911133,
+      "kl": 0.759245615452528,
+      "learning_rate": 9.632344625496255e-07,
+      "loss": 0.0304,
+      "num_tokens": 5696520.0,
+      "reward": 0.767822265625,
+      "reward_std": 0.011308575049042702,
+      "rewards//mean": 0.767822265625,
+      "rewards//std": 0.016541730612516403,
+      "step": 659
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.132,
+      "grad_norm": 5.445815086364746,
+      "kl": 0.3996684094890952,
+      "learning_rate": 9.63114934392728e-07,
+      "loss": 0.016,
+      "num_tokens": 5705160.0,
+      "reward": 0.7298583984375,
+      "reward_std": 0.005815407261252403,
+      "rewards//mean": 0.7298583984375,
+      "rewards//std": 0.02091708965599537,
+      "step": 660
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1322,
+      "grad_norm": 5.452217102050781,
+      "kl": 0.6336718033999205,
+      "learning_rate": 9.6299521969319e-07,
+      "loss": 0.0253,
+      "num_tokens": 5713848.0,
+      "reward": 0.77862548828125,
+      "reward_std": 0.00826502125710249,
+      "rewards//mean": 0.77862548828125,
+      "rewards//std": 0.017908191308379173,
+      "step": 661
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1324,
+      "grad_norm": 6.652682304382324,
+      "kl": 0.6581048853695393,
+      "learning_rate": 9.628753184992333e-07,
+      "loss": 0.0263,
+      "num_tokens": 5722440.0,
+      "reward": 0.77764892578125,
+      "reward_std": 0.013120852410793304,
+      "rewards//mean": 0.77764892578125,
+      "rewards//std": 0.02943238988518715,
+      "step": 662
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1326,
+      "grad_norm": 5.127926349639893,
+      "kl": 0.7042171955108643,
+      "learning_rate": 9.627552308591533e-07,
+      "loss": 0.0282,
+      "num_tokens": 5731056.0,
+      "reward": 0.7740478515625,
+      "reward_std": 0.008530810475349426,
+      "rewards//mean": 0.7740478515625,
+      "rewards//std": 0.02753545716404915,
+      "step": 663
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1328,
+      "grad_norm": 5.415334701538086,
+      "kl": 0.842731736600399,
+      "learning_rate": 9.62634956821322e-07,
+      "loss": 0.0337,
+      "num_tokens": 5739704.0,
+      "reward": 0.73822021484375,
+      "reward_std": 0.009494063444435596,
+      "rewards//mean": 0.73822021484375,
+      "rewards//std": 0.0287428367882967,
+      "step": 664
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.133,
+      "grad_norm": 6.197539329528809,
+      "kl": 0.8267682623118162,
+      "learning_rate": 9.625144964341852e-07,
+      "loss": 0.0331,
+      "num_tokens": 5748272.0,
+      "reward": 0.759521484375,
+      "reward_std": 0.011234431527554989,
+      "rewards//mean": 0.759521484375,
+      "rewards//std": 0.027664074674248695,
+      "step": 665
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1332,
+      "grad_norm": 5.348916530609131,
+      "kl": 0.7823239136487246,
+      "learning_rate": 9.623938497462645e-07,
+      "loss": 0.0313,
+      "num_tokens": 5756944.0,
+      "reward": 0.7764892578125,
+      "reward_std": 0.009534705430269241,
+      "rewards//mean": 0.7764892578125,
+      "rewards//std": 0.024204833433032036,
+      "step": 666
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1334,
+      "grad_norm": 8.233922958374023,
+      "kl": 1.2101830318570137,
+      "learning_rate": 9.622730168061567e-07,
+      "loss": 0.0484,
+      "num_tokens": 5765464.0,
+      "reward": 0.74365234375,
+      "reward_std": 0.012155747041106224,
+      "rewards//mean": 0.74365234375,
+      "rewards//std": 0.028403842821717262,
+      "step": 667
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1336,
+      "grad_norm": 7.8597798347473145,
+      "kl": 0.7464694883674383,
+      "learning_rate": 9.621519976625326e-07,
+      "loss": 0.0299,
+      "num_tokens": 5774152.0,
+      "reward": 0.7633056640625,
+      "reward_std": 0.012880093418061733,
+      "rewards//mean": 0.7633056640625,
+      "rewards//std": 0.028837284073233604,
+      "step": 668
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1338,
+      "grad_norm": 6.381985187530518,
+      "kl": 0.6748458743095398,
+      "learning_rate": 9.620307923641392e-07,
+      "loss": 0.027,
+      "num_tokens": 5782856.0,
+      "reward": 0.75762939453125,
+      "reward_std": 0.00775592215359211,
+      "rewards//mean": 0.75762939453125,
+      "rewards//std": 0.029168905690312386,
+      "step": 669
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.134,
+      "grad_norm": 4.677013874053955,
+      "kl": 1.0138772446662188,
+      "learning_rate": 9.61909400959798e-07,
+      "loss": 0.0406,
+      "num_tokens": 5791512.0,
+      "reward": 0.7457275390625,
+      "reward_std": 0.010004445910453796,
+      "rewards//mean": 0.7457275390625,
+      "rewards//std": 0.03177051991224289,
+      "step": 670
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1342,
+      "grad_norm": 7.821444034576416,
+      "kl": 1.091262711212039,
+      "learning_rate": 9.617878234984054e-07,
+      "loss": 0.0437,
+      "num_tokens": 5800024.0,
+      "reward": 0.7340087890625,
+      "reward_std": 0.011750814504921436,
+      "rewards//mean": 0.7340087890625,
+      "rewards//std": 0.023844420909881592,
+      "step": 671
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1344,
+      "grad_norm": 5.484076023101807,
+      "kl": 0.47971958946436644,
+      "learning_rate": 9.616660600289327e-07,
+      "loss": 0.0192,
+      "num_tokens": 5808632.0,
+      "reward": 0.7623291015625,
+      "reward_std": 0.009718211367726326,
+      "rewards//mean": 0.7623291015625,
+      "rewards//std": 0.02152203768491745,
+      "step": 672
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1346,
+      "grad_norm": 6.548192977905273,
+      "kl": 0.6700717397034168,
+      "learning_rate": 9.615441106004262e-07,
+      "loss": 0.0268,
+      "num_tokens": 5817208.0,
+      "reward": 0.75799560546875,
+      "reward_std": 0.009721084497869015,
+      "rewards//mean": 0.75799560546875,
+      "rewards//std": 0.02633235976099968,
+      "step": 673
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1348,
+      "grad_norm": 8.142129898071289,
+      "kl": 1.1573685612529516,
+      "learning_rate": 9.614219752620072e-07,
+      "loss": 0.0463,
+      "num_tokens": 5826000.0,
+      "reward": 0.76007080078125,
+      "reward_std": 0.013684765435755253,
+      "rewards//mean": 0.76007080078125,
+      "rewards//std": 0.026234449818730354,
+      "step": 674
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.135,
+      "grad_norm": 10.413251876831055,
+      "kl": 0.9288819264620543,
+      "learning_rate": 9.612996540628717e-07,
+      "loss": 0.0372,
+      "num_tokens": 5834584.0,
+      "reward": 0.75897216796875,
+      "reward_std": 0.013641918078064919,
+      "rewards//mean": 0.75897216796875,
+      "rewards//std": 0.02662734128534794,
+      "step": 675
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1352,
+      "grad_norm": 6.3422393798828125,
+      "kl": 0.916122330352664,
+      "learning_rate": 9.611771470522907e-07,
+      "loss": 0.0366,
+      "num_tokens": 5843224.0,
+      "reward": 0.729736328125,
+      "reward_std": 0.010156441479921341,
+      "rewards//mean": 0.729736328125,
+      "rewards//std": 0.021630434319376945,
+      "step": 676
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1354,
+      "grad_norm": 6.724777698516846,
+      "kl": 0.9932534657418728,
+      "learning_rate": 9.6105445427961e-07,
+      "loss": 0.0397,
+      "num_tokens": 5851800.0,
+      "reward": 0.7030029296875,
+      "reward_std": 0.010675765573978424,
+      "rewards//mean": 0.7030029296875,
+      "rewards//std": 0.02326083369553089,
+      "step": 677
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1356,
+      "grad_norm": 5.712681770324707,
+      "kl": 1.0585048235952854,
+      "learning_rate": 9.609315757942502e-07,
+      "loss": 0.0423,
+      "num_tokens": 5860408.0,
+      "reward": 0.743896484375,
+      "reward_std": 0.00818407628685236,
+      "rewards//mean": 0.743896484375,
+      "rewards//std": 0.0249673780053854,
+      "step": 678
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1358,
+      "grad_norm": 5.611710071563721,
+      "kl": 0.8865110967308283,
+      "learning_rate": 9.608085116457068e-07,
+      "loss": 0.0355,
+      "num_tokens": 5869048.0,
+      "reward": 0.760498046875,
+      "reward_std": 0.008227774873375893,
+      "rewards//mean": 0.760498046875,
+      "rewards//std": 0.02861093543469906,
+      "step": 679
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.136,
+      "grad_norm": 7.391247749328613,
+      "kl": 1.793531896546483,
+      "learning_rate": 9.606852618835502e-07,
+      "loss": 0.0717,
+      "num_tokens": 5877704.0,
+      "reward": 0.76129150390625,
+      "reward_std": 0.013443115167319775,
+      "rewards//mean": 0.76129150390625,
+      "rewards//std": 0.029619522392749786,
+      "step": 680
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1362,
+      "grad_norm": 5.805166721343994,
+      "kl": 1.892662312835455,
+      "learning_rate": 9.60561826557425e-07,
+      "loss": 0.0757,
+      "num_tokens": 5886232.0,
+      "reward": 0.743408203125,
+      "reward_std": 0.009377227164804935,
+      "rewards//mean": 0.743408203125,
+      "rewards//std": 0.02087833546102047,
+      "step": 681
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1364,
+      "grad_norm": 6.622632026672363,
+      "kl": 1.2632559463381767,
+      "learning_rate": 9.604382057170512e-07,
+      "loss": 0.0505,
+      "num_tokens": 5894872.0,
+      "reward": 0.732421875,
+      "reward_std": 0.01137983426451683,
+      "rewards//mean": 0.732421875,
+      "rewards//std": 0.036037378013134,
+      "step": 682
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1366,
+      "grad_norm": 9.4186372756958,
+      "kl": 1.7058883737772703,
+      "learning_rate": 9.603143994122232e-07,
+      "loss": 0.0682,
+      "num_tokens": 5903480.0,
+      "reward": 0.7294921875,
+      "reward_std": 0.015523059293627739,
+      "rewards//mean": 0.7294921875,
+      "rewards//std": 0.03373950347304344,
+      "step": 683
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1368,
+      "grad_norm": 6.104360103607178,
+      "kl": 1.9442303087562323,
+      "learning_rate": 9.601904076928102e-07,
+      "loss": 0.0778,
+      "num_tokens": 5912088.0,
+      "reward": 0.7520751953125,
+      "reward_std": 0.009610344655811787,
+      "rewards//mean": 0.7520751953125,
+      "rewards//std": 0.027347903698682785,
+      "step": 684
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.137,
+      "grad_norm": 7.810328006744385,
+      "kl": 1.7898661009967327,
+      "learning_rate": 9.60066230608756e-07,
+      "loss": 0.0716,
+      "num_tokens": 5920760.0,
+      "reward": 0.78533935546875,
+      "reward_std": 0.011764805763959885,
+      "rewards//mean": 0.78533935546875,
+      "rewards//std": 0.020439328625798225,
+      "step": 685
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1372,
+      "grad_norm": 8.834696769714355,
+      "kl": 2.512271039187908,
+      "learning_rate": 9.599418682100792e-07,
+      "loss": 0.1005,
+      "num_tokens": 5929320.0,
+      "reward": 0.7481689453125,
+      "reward_std": 0.009096116758883,
+      "rewards//mean": 0.7481689453125,
+      "rewards//std": 0.02685638517141342,
+      "step": 686
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1374,
+      "grad_norm": 5.598320960998535,
+      "kl": 1.4660762287676334,
+      "learning_rate": 9.598173205468727e-07,
+      "loss": 0.0586,
+      "num_tokens": 5937912.0,
+      "reward": 0.7310791015625,
+      "reward_std": 0.009189204312860966,
+      "rewards//mean": 0.7310791015625,
+      "rewards//std": 0.031467948108911514,
+      "step": 687
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1376,
+      "grad_norm": 7.345466136932373,
+      "kl": 1.1756226010620594,
+      "learning_rate": 9.596925876693047e-07,
+      "loss": 0.047,
+      "num_tokens": 5946624.0,
+      "reward": 0.7484130859375,
+      "reward_std": 0.011727170087397099,
+      "rewards//mean": 0.7484130859375,
+      "rewards//std": 0.027522260323166847,
+      "step": 688
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1378,
+      "grad_norm": 5.2736382484436035,
+      "kl": 2.056324951350689,
+      "learning_rate": 9.595676696276171e-07,
+      "loss": 0.0823,
+      "num_tokens": 5955192.0,
+      "reward": 0.71044921875,
+      "reward_std": 0.011874396353960037,
+      "rewards//mean": 0.71044921875,
+      "rewards//std": 0.032641246914863586,
+      "step": 689
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.138,
+      "grad_norm": 6.256668567657471,
+      "kl": 1.3164225164800882,
+      "learning_rate": 9.594425664721274e-07,
+      "loss": 0.0527,
+      "num_tokens": 5963760.0,
+      "reward": 0.76806640625,
+      "reward_std": 0.011647808365523815,
+      "rewards//mean": 0.76806640625,
+      "rewards//std": 0.023401303216814995,
+      "step": 690
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1382,
+      "grad_norm": 6.807124614715576,
+      "kl": 1.285219006240368,
+      "learning_rate": 9.593172782532267e-07,
+      "loss": 0.0514,
+      "num_tokens": 5972352.0,
+      "reward": 0.7537841796875,
+      "reward_std": 0.01029349397867918,
+      "rewards//mean": 0.7537841796875,
+      "rewards//std": 0.029031902551651,
+      "step": 691
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1384,
+      "grad_norm": 8.663215637207031,
+      "kl": 2.398256568238139,
+      "learning_rate": 9.591918050213813e-07,
+      "loss": 0.0959,
+      "num_tokens": 5980920.0,
+      "reward": 0.74505615234375,
+      "reward_std": 0.011829331517219543,
+      "rewards//mean": 0.74505615234375,
+      "rewards//std": 0.026759473606944084,
+      "step": 692
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1386,
+      "grad_norm": 6.38392448425293,
+      "kl": 1.77195187471807,
+      "learning_rate": 9.590661468271318e-07,
+      "loss": 0.0709,
+      "num_tokens": 5989576.0,
+      "reward": 0.759765625,
+      "reward_std": 0.011866278946399689,
+      "rewards//mean": 0.759765625,
+      "rewards//std": 0.03202323615550995,
+      "step": 693
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1388,
+      "grad_norm": 6.773484706878662,
+      "kl": 1.5403836891055107,
+      "learning_rate": 9.589403037210931e-07,
+      "loss": 0.0616,
+      "num_tokens": 5998160.0,
+      "reward": 0.7611083984375,
+      "reward_std": 0.00890538189560175,
+      "rewards//mean": 0.7611083984375,
+      "rewards//std": 0.023263435810804367,
+      "step": 694
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.139,
+      "grad_norm": 8.277812004089355,
+      "kl": 3.2806071285158396,
+      "learning_rate": 9.58814275753955e-07,
+      "loss": 0.1312,
+      "num_tokens": 6006776.0,
+      "reward": 0.7581787109375,
+      "reward_std": 0.013688182458281517,
+      "rewards//mean": 0.7581787109375,
+      "rewards//std": 0.03678056225180626,
+      "step": 695
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1392,
+      "grad_norm": 8.247855186462402,
+      "kl": 0.9284660518169403,
+      "learning_rate": 9.586880629764817e-07,
+      "loss": 0.0371,
+      "num_tokens": 6015376.0,
+      "reward": 0.77325439453125,
+      "reward_std": 0.009077081456780434,
+      "rewards//mean": 0.77325439453125,
+      "rewards//std": 0.018577009439468384,
+      "step": 696
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1394,
+      "grad_norm": 6.369467258453369,
+      "kl": 2.8970567900687456,
+      "learning_rate": 9.585616654395112e-07,
+      "loss": 0.1159,
+      "num_tokens": 6023976.0,
+      "reward": 0.75634765625,
+      "reward_std": 0.01232706569135189,
+      "rewards//mean": 0.75634765625,
+      "rewards//std": 0.0297206062823534,
+      "step": 697
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1396,
+      "grad_norm": 5.498188018798828,
+      "kl": 2.0883035007864237,
+      "learning_rate": 9.584350831939569e-07,
+      "loss": 0.0835,
+      "num_tokens": 6032584.0,
+      "reward": 0.78399658203125,
+      "reward_std": 0.01031227596104145,
+      "rewards//mean": 0.78399658203125,
+      "rewards//std": 0.023454852402210236,
+      "step": 698
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1398,
+      "grad_norm": 5.672773361206055,
+      "kl": 1.3796262983232737,
+      "learning_rate": 9.58308316290806e-07,
+      "loss": 0.0552,
+      "num_tokens": 6041200.0,
+      "reward": 0.73748779296875,
+      "reward_std": 0.007399318739771843,
+      "rewards//mean": 0.73748779296875,
+      "rewards//std": 0.034528397023677826,
+      "step": 699
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.14,
+      "grad_norm": 4.546067237854004,
+      "kl": 0.89860107190907,
+      "learning_rate": 9.581813647811197e-07,
+      "loss": 0.0359,
+      "num_tokens": 6049776.0,
+      "reward": 0.7552490234375,
+      "reward_std": 0.00793510116636753,
+      "rewards//mean": 0.7552490234375,
+      "rewards//std": 0.03077531047165394,
+      "step": 700
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1402,
+      "grad_norm": 6.4733734130859375,
+      "kl": 2.410829773172736,
+      "learning_rate": 9.580542287160346e-07,
+      "loss": 0.0964,
+      "num_tokens": 6058352.0,
+      "reward": 0.77392578125,
+      "reward_std": 0.012427553534507751,
+      "rewards//mean": 0.77392578125,
+      "rewards//std": 0.03509419038891792,
+      "step": 701
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1404,
+      "grad_norm": 6.268716335296631,
+      "kl": 1.872730614617467,
+      "learning_rate": 9.579269081467613e-07,
+      "loss": 0.0749,
+      "num_tokens": 6066912.0,
+      "reward": 0.74884033203125,
+      "reward_std": 0.010377561673521996,
+      "rewards//mean": 0.74884033203125,
+      "rewards//std": 0.027583450078964233,
+      "step": 702
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1406,
+      "grad_norm": 6.586522102355957,
+      "kl": 1.6278119310736656,
+      "learning_rate": 9.57799403124584e-07,
+      "loss": 0.0651,
+      "num_tokens": 6075536.0,
+      "reward": 0.76214599609375,
+      "reward_std": 0.012576766312122345,
+      "rewards//mean": 0.76214599609375,
+      "rewards//std": 0.029373183846473694,
+      "step": 703
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1408,
+      "grad_norm": 4.88497257232666,
+      "kl": 1.7092098165303469,
+      "learning_rate": 9.576717137008617e-07,
+      "loss": 0.0684,
+      "num_tokens": 6084152.0,
+      "reward": 0.7347412109375,
+      "reward_std": 0.009226744994521141,
+      "rewards//mean": 0.7347412109375,
+      "rewards//std": 0.03108462318778038,
+      "step": 704
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.141,
+      "grad_norm": 4.4319233894348145,
+      "kl": 0.4497374640777707,
+      "learning_rate": 9.575438399270278e-07,
+      "loss": 0.018,
+      "num_tokens": 6092728.0,
+      "reward": 0.7845458984375,
+      "reward_std": 0.0051748850382864475,
+      "rewards//mean": 0.7845458984375,
+      "rewards//std": 0.017977897077798843,
+      "step": 705
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1412,
+      "grad_norm": 4.941458702087402,
+      "kl": 1.4851972945034504,
+      "learning_rate": 9.5741578185459e-07,
+      "loss": 0.0594,
+      "num_tokens": 6101480.0,
+      "reward": 0.72454833984375,
+      "reward_std": 0.010255906730890274,
+      "rewards//mean": 0.72454833984375,
+      "rewards//std": 0.03637181594967842,
+      "step": 706
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1414,
+      "grad_norm": 8.730281829833984,
+      "kl": 4.643188184127212,
+      "learning_rate": 9.572875395351301e-07,
+      "loss": 0.1857,
+      "num_tokens": 6110208.0,
+      "reward": 0.74664306640625,
+      "reward_std": 0.019012529402971268,
+      "rewards//mean": 0.74664306640625,
+      "rewards//std": 0.03865654394030571,
+      "step": 707
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1416,
+      "grad_norm": 5.186831951141357,
+      "kl": 2.2443429697304964,
+      "learning_rate": 9.571591130203037e-07,
+      "loss": 0.0898,
+      "num_tokens": 6118832.0,
+      "reward": 0.74261474609375,
+      "reward_std": 0.010612553916871548,
+      "rewards//mean": 0.74261474609375,
+      "rewards//std": 0.02681259624660015,
+      "step": 708
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1418,
+      "grad_norm": 6.91388463973999,
+      "kl": 1.2408266803249717,
+      "learning_rate": 9.570305023618415e-07,
+      "loss": 0.0496,
+      "num_tokens": 6127480.0,
+      "reward": 0.749755859375,
+      "reward_std": 0.008738374337553978,
+      "rewards//mean": 0.749755859375,
+      "rewards//std": 0.03242814913392067,
+      "step": 709
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.142,
+      "grad_norm": 5.295790672302246,
+      "kl": 2.0118208341300488,
+      "learning_rate": 9.569017076115475e-07,
+      "loss": 0.0805,
+      "num_tokens": 6136016.0,
+      "reward": 0.7340087890625,
+      "reward_std": 0.01210230216383934,
+      "rewards//mean": 0.7340087890625,
+      "rewards//std": 0.028114309534430504,
+      "step": 710
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1422,
+      "grad_norm": 5.204315662384033,
+      "kl": 1.6005570348352194,
+      "learning_rate": 9.567727288213004e-07,
+      "loss": 0.064,
+      "num_tokens": 6144688.0,
+      "reward": 0.75543212890625,
+      "reward_std": 0.008108418434858322,
+      "rewards//mean": 0.75543212890625,
+      "rewards//std": 0.02097950503230095,
+      "step": 711
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1424,
+      "grad_norm": 6.038320064544678,
+      "kl": 2.145377606153488,
+      "learning_rate": 9.566435660430527e-07,
+      "loss": 0.0858,
+      "num_tokens": 6153288.0,
+      "reward": 0.74969482421875,
+      "reward_std": 0.005925554782152176,
+      "rewards//mean": 0.74969482421875,
+      "rewards//std": 0.03265463560819626,
+      "step": 712
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1426,
+      "grad_norm": 6.113224983215332,
+      "kl": 1.8366315010935068,
+      "learning_rate": 9.565142193288312e-07,
+      "loss": 0.0735,
+      "num_tokens": 6161936.0,
+      "reward": 0.75714111328125,
+      "reward_std": 0.011059759184718132,
+      "rewards//mean": 0.75714111328125,
+      "rewards//std": 0.03809284418821335,
+      "step": 713
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1428,
+      "grad_norm": 16.415096282958984,
+      "kl": 6.374369312077761,
+      "learning_rate": 9.563846887307368e-07,
+      "loss": 0.255,
+      "num_tokens": 6170696.0,
+      "reward": 0.76641845703125,
+      "reward_std": 0.01262245699763298,
+      "rewards//mean": 0.76641845703125,
+      "rewards//std": 0.029513539746403694,
+      "step": 714
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.143,
+      "grad_norm": 3.9718143939971924,
+      "kl": 1.2525438275188208,
+      "learning_rate": 9.562549743009442e-07,
+      "loss": 0.0501,
+      "num_tokens": 6179384.0,
+      "reward": 0.776123046875,
+      "reward_std": 0.006306346971541643,
+      "rewards//mean": 0.776123046875,
+      "rewards//std": 0.025131750851869583,
+      "step": 715
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1432,
+      "grad_norm": 5.382948398590088,
+      "kl": 3.109356554225087,
+      "learning_rate": 9.561250760917025e-07,
+      "loss": 0.1244,
+      "num_tokens": 6187896.0,
+      "reward": 0.76251220703125,
+      "reward_std": 0.010493282228708267,
+      "rewards//mean": 0.76251220703125,
+      "rewards//std": 0.025067143142223358,
+      "step": 716
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1434,
+      "grad_norm": 5.200241565704346,
+      "kl": 1.0269698034971952,
+      "learning_rate": 9.55994994155335e-07,
+      "loss": 0.0411,
+      "num_tokens": 6196536.0,
+      "reward": 0.7301025390625,
+      "reward_std": 0.006362422835081816,
+      "rewards//mean": 0.7301025390625,
+      "rewards//std": 0.019817089661955833,
+      "step": 717
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1436,
+      "grad_norm": 5.346518516540527,
+      "kl": 1.3138235844671726,
+      "learning_rate": 9.558647285442381e-07,
+      "loss": 0.0526,
+      "num_tokens": 6205168.0,
+      "reward": 0.7586669921875,
+      "reward_std": 0.007854700088500977,
+      "rewards//mean": 0.7586669921875,
+      "rewards//std": 0.016418185085058212,
+      "step": 718
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1438,
+      "grad_norm": 14.749935150146484,
+      "kl": 4.530792691744864,
+      "learning_rate": 9.55734279310883e-07,
+      "loss": 0.1812,
+      "num_tokens": 6213784.0,
+      "reward": 0.7567138671875,
+      "reward_std": 0.008137339726090431,
+      "rewards//mean": 0.7567138671875,
+      "rewards//std": 0.02853119745850563,
+      "step": 719
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.144,
+      "grad_norm": 6.514327526092529,
+      "kl": 2.0024897698313,
+      "learning_rate": 9.55603646507815e-07,
+      "loss": 0.0801,
+      "num_tokens": 6222432.0,
+      "reward": 0.754638671875,
+      "reward_std": 0.008971983566880226,
+      "rewards//mean": 0.754638671875,
+      "rewards//std": 0.026665473356842995,
+      "step": 720
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1442,
+      "grad_norm": 4.103392601013184,
+      "kl": 3.0026297997683287,
+      "learning_rate": 9.554728301876524e-07,
+      "loss": 0.1201,
+      "num_tokens": 6231232.0,
+      "reward": 0.75634765625,
+      "reward_std": 0.011162412352859974,
+      "rewards//mean": 0.75634765625,
+      "rewards//std": 0.03155466169118881,
+      "step": 721
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1444,
+      "grad_norm": 4.249013900756836,
+      "kl": 1.2251688614487648,
+      "learning_rate": 9.553418304030885e-07,
+      "loss": 0.049,
+      "num_tokens": 6239984.0,
+      "reward": 0.7470703125,
+      "reward_std": 0.00776326097548008,
+      "rewards//mean": 0.7470703125,
+      "rewards//std": 0.02208612486720085,
+      "step": 722
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1446,
+      "grad_norm": 3.8857667446136475,
+      "kl": 1.779951322823763,
+      "learning_rate": 9.552106472068897e-07,
+      "loss": 0.0712,
+      "num_tokens": 6248576.0,
+      "reward": 0.74786376953125,
+      "reward_std": 0.008643015287816525,
+      "rewards//mean": 0.74786376953125,
+      "rewards//std": 0.02321873977780342,
+      "step": 723
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1448,
+      "grad_norm": 9.575364112854004,
+      "kl": 3.3338482324033976,
+      "learning_rate": 9.550792806518967e-07,
+      "loss": 0.1334,
+      "num_tokens": 6257128.0,
+      "reward": 0.7659912109375,
+      "reward_std": 0.007942140102386475,
+      "rewards//mean": 0.7659912109375,
+      "rewards//std": 0.022742237895727158,
+      "step": 724
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.145,
+      "grad_norm": 8.159613609313965,
+      "kl": 1.2658102735877037,
+      "learning_rate": 9.549477307910236e-07,
+      "loss": 0.0506,
+      "num_tokens": 6265768.0,
+      "reward": 0.782470703125,
+      "reward_std": 0.007177484221756458,
+      "rewards//mean": 0.782470703125,
+      "rewards//std": 0.02122350223362446,
+      "step": 725
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1452,
+      "grad_norm": 4.528134346008301,
+      "kl": 1.4327340200543404,
+      "learning_rate": 9.548159976772592e-07,
+      "loss": 0.0573,
+      "num_tokens": 6274448.0,
+      "reward": 0.767333984375,
+      "reward_std": 0.007283302955329418,
+      "rewards//mean": 0.767333984375,
+      "rewards//std": 0.01673099398612976,
+      "step": 726
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1454,
+      "grad_norm": 10.091915130615234,
+      "kl": 2.697010073810816,
+      "learning_rate": 9.546840813636652e-07,
+      "loss": 0.1079,
+      "num_tokens": 6283040.0,
+      "reward": 0.74798583984375,
+      "reward_std": 0.007095325272530317,
+      "rewards//mean": 0.74798583984375,
+      "rewards//std": 0.02330269105732441,
+      "step": 727
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1456,
+      "grad_norm": 7.179139614105225,
+      "kl": 1.4060106426477432,
+      "learning_rate": 9.545519819033777e-07,
+      "loss": 0.0562,
+      "num_tokens": 6291704.0,
+      "reward": 0.72039794921875,
+      "reward_std": 0.011540853418409824,
+      "rewards//mean": 0.72039794921875,
+      "rewards//std": 0.038297347724437714,
+      "step": 728
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1458,
+      "grad_norm": 4.981100082397461,
+      "kl": 1.1590757528319955,
+      "learning_rate": 9.544196993496062e-07,
+      "loss": 0.0464,
+      "num_tokens": 6300344.0,
+      "reward": 0.759521484375,
+      "reward_std": 0.006217144895344973,
+      "rewards//mean": 0.759521484375,
+      "rewards//std": 0.032910048961639404,
+      "step": 729
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.146,
+      "grad_norm": 6.2702717781066895,
+      "kl": 2.595333404839039,
+      "learning_rate": 9.54287233755634e-07,
+      "loss": 0.1038,
+      "num_tokens": 6308912.0,
+      "reward": 0.75537109375,
+      "reward_std": 0.014401756227016449,
+      "rewards//mean": 0.75537109375,
+      "rewards//std": 0.029078010469675064,
+      "step": 730
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1462,
+      "grad_norm": 3.8329060077667236,
+      "kl": 1.225698129273951,
+      "learning_rate": 9.541545851748185e-07,
+      "loss": 0.049,
+      "num_tokens": 6317680.0,
+      "reward": 0.7335205078125,
+      "reward_std": 0.006505907978862524,
+      "rewards//mean": 0.7335205078125,
+      "rewards//std": 0.029027730226516724,
+      "step": 731
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1464,
+      "grad_norm": 4.659354209899902,
+      "kl": 0.8769095670431852,
+      "learning_rate": 9.540217536605905e-07,
+      "loss": 0.0351,
+      "num_tokens": 6326368.0,
+      "reward": 0.802001953125,
+      "reward_std": 0.00857323594391346,
+      "rewards//mean": 0.802001953125,
+      "rewards//std": 0.016875134781003,
+      "step": 732
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1466,
+      "grad_norm": 3.9992916584014893,
+      "kl": 2.298052353784442,
+      "learning_rate": 9.538887392664543e-07,
+      "loss": 0.0919,
+      "num_tokens": 6335016.0,
+      "reward": 0.7589111328125,
+      "reward_std": 0.010746128857135773,
+      "rewards//mean": 0.7589111328125,
+      "rewards//std": 0.035307276993989944,
+      "step": 733
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1468,
+      "grad_norm": 5.744341850280762,
+      "kl": 0.4098111856728792,
+      "learning_rate": 9.537555420459881e-07,
+      "loss": 0.0164,
+      "num_tokens": 6343744.0,
+      "reward": 0.77294921875,
+      "reward_std": 0.006228247657418251,
+      "rewards//mean": 0.77294921875,
+      "rewards//std": 0.02306772582232952,
+      "step": 734
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.147,
+      "grad_norm": 3.447507619857788,
+      "kl": 0.36606572940945625,
+      "learning_rate": 9.53622162052844e-07,
+      "loss": 0.0146,
+      "num_tokens": 6352456.0,
+      "reward": 0.78509521484375,
+      "reward_std": 0.005339228548109531,
+      "rewards//mean": 0.78509521484375,
+      "rewards//std": 0.01976536586880684,
+      "step": 735
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1472,
+      "grad_norm": 5.920298099517822,
+      "kl": 3.188815062865615,
+      "learning_rate": 9.534885993407474e-07,
+      "loss": 0.1276,
+      "num_tokens": 6361208.0,
+      "reward": 0.765869140625,
+      "reward_std": 0.007528609596192837,
+      "rewards//mean": 0.765869140625,
+      "rewards//std": 0.014346149750053883,
+      "step": 736
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1474,
+      "grad_norm": 3.670410633087158,
+      "kl": 1.228419966995716,
+      "learning_rate": 9.53354853963497e-07,
+      "loss": 0.0491,
+      "num_tokens": 6369800.0,
+      "reward": 0.7642822265625,
+      "reward_std": 0.008165386505424976,
+      "rewards//mean": 0.7642822265625,
+      "rewards//std": 0.021476976573467255,
+      "step": 737
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1476,
+      "grad_norm": 5.1945343017578125,
+      "kl": 1.1253601480275393,
+      "learning_rate": 9.532209259749658e-07,
+      "loss": 0.045,
+      "num_tokens": 6378480.0,
+      "reward": 0.75177001953125,
+      "reward_std": 0.010367737151682377,
+      "rewards//mean": 0.75177001953125,
+      "rewards//std": 0.02931385673582554,
+      "step": 738
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1478,
+      "grad_norm": 5.994574546813965,
+      "kl": 1.9060461390763521,
+      "learning_rate": 9.530868154290996e-07,
+      "loss": 0.0762,
+      "num_tokens": 6387056.0,
+      "reward": 0.73992919921875,
+      "reward_std": 0.010355432517826557,
+      "rewards//mean": 0.73992919921875,
+      "rewards//std": 0.04093245416879654,
+      "step": 739
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.148,
+      "grad_norm": 4.982955455780029,
+      "kl": 0.44224715419113636,
+      "learning_rate": 9.529525223799184e-07,
+      "loss": 0.0177,
+      "num_tokens": 6395720.0,
+      "reward": 0.78619384765625,
+      "reward_std": 0.006468282546848059,
+      "rewards//mean": 0.78619384765625,
+      "rewards//std": 0.018999995663762093,
+      "step": 740
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1482,
+      "grad_norm": 3.5400283336639404,
+      "kl": 0.6346832923591137,
+      "learning_rate": 9.528180468815154e-07,
+      "loss": 0.0254,
+      "num_tokens": 6404408.0,
+      "reward": 0.76141357421875,
+      "reward_std": 0.0055562881752848625,
+      "rewards//mean": 0.76141357421875,
+      "rewards//std": 0.017542673274874687,
+      "step": 741
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1484,
+      "grad_norm": 6.327049255371094,
+      "kl": 2.22835841961205,
+      "learning_rate": 9.526833889880572e-07,
+      "loss": 0.0891,
+      "num_tokens": 6413088.0,
+      "reward": 0.76513671875,
+      "reward_std": 0.008300656452775002,
+      "rewards//mean": 0.76513671875,
+      "rewards//std": 0.027826661244034767,
+      "step": 742
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1486,
+      "grad_norm": 4.479706764221191,
+      "kl": 2.2920671328902245,
+      "learning_rate": 9.525485487537841e-07,
+      "loss": 0.0917,
+      "num_tokens": 6421912.0,
+      "reward": 0.7872314453125,
+      "reward_std": 0.00978466309607029,
+      "rewards//mean": 0.7872314453125,
+      "rewards//std": 0.021743163466453552,
+      "step": 743
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1488,
+      "grad_norm": 4.605365753173828,
+      "kl": 0.556102741509676,
+      "learning_rate": 9.524135262330098e-07,
+      "loss": 0.0222,
+      "num_tokens": 6430552.0,
+      "reward": 0.7945556640625,
+      "reward_std": 0.006804032251238823,
+      "rewards//mean": 0.7945556640625,
+      "rewards//std": 0.018567737191915512,
+      "step": 744
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.149,
+      "grad_norm": 3.1205813884735107,
+      "kl": 0.6468249522149563,
+      "learning_rate": 9.522783214801211e-07,
+      "loss": 0.0259,
+      "num_tokens": 6439144.0,
+      "reward": 0.77020263671875,
+      "reward_std": 0.00491624278947711,
+      "rewards//mean": 0.77020263671875,
+      "rewards//std": 0.013160609640181065,
+      "step": 745
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1492,
+      "grad_norm": 5.286034107208252,
+      "kl": 0.9488318208605051,
+      "learning_rate": 9.521429345495786e-07,
+      "loss": 0.038,
+      "num_tokens": 6447728.0,
+      "reward": 0.779296875,
+      "reward_std": 0.006964164320379496,
+      "rewards//mean": 0.779296875,
+      "rewards//std": 0.014566050842404366,
+      "step": 746
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1494,
+      "grad_norm": 7.0882344245910645,
+      "kl": 1.0645874477922916,
+      "learning_rate": 9.520073654959162e-07,
+      "loss": 0.0426,
+      "num_tokens": 6456312.0,
+      "reward": 0.771484375,
+      "reward_std": 0.007842430844902992,
+      "rewards//mean": 0.771484375,
+      "rewards//std": 0.029090501368045807,
+      "step": 747
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1496,
+      "grad_norm": 4.138265609741211,
+      "kl": 1.7977023757994175,
+      "learning_rate": 9.518716143737409e-07,
+      "loss": 0.0719,
+      "num_tokens": 6464936.0,
+      "reward": 0.78375244140625,
+      "reward_std": 0.010723061859607697,
+      "rewards//mean": 0.78375244140625,
+      "rewards//std": 0.019077913835644722,
+      "step": 748
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1498,
+      "grad_norm": 4.277359485626221,
+      "kl": 2.4151702784001827,
+      "learning_rate": 9.517356812377335e-07,
+      "loss": 0.0966,
+      "num_tokens": 6473664.0,
+      "reward": 0.734619140625,
+      "reward_std": 0.008682307787239552,
+      "rewards//mean": 0.734619140625,
+      "rewards//std": 0.03941971808671951,
+      "step": 749
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.15,
+      "grad_norm": 5.214427471160889,
+      "kl": 2.4279807303100824,
+      "learning_rate": 9.515995661426477e-07,
+      "loss": 0.0971,
+      "num_tokens": 6482200.0,
+      "reward": 0.74639892578125,
+      "reward_std": 0.010630439035594463,
+      "rewards//mean": 0.74639892578125,
+      "rewards//std": 0.02370777539908886,
+      "step": 750
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1502,
+      "grad_norm": 4.322005271911621,
+      "kl": 2.101282015442848,
+      "learning_rate": 9.514632691433106e-07,
+      "loss": 0.0841,
+      "num_tokens": 6490712.0,
+      "reward": 0.751953125,
+      "reward_std": 0.007668613456189632,
+      "rewards//mean": 0.751953125,
+      "rewards//std": 0.028262794017791748,
+      "step": 751
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1504,
+      "grad_norm": 7.709218502044678,
+      "kl": 2.939092857763171,
+      "learning_rate": 9.513267902946227e-07,
+      "loss": 0.1176,
+      "num_tokens": 6499392.0,
+      "reward": 0.77880859375,
+      "reward_std": 0.01251686830073595,
+      "rewards//mean": 0.77880859375,
+      "rewards//std": 0.024384593591094017,
+      "step": 752
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1506,
+      "grad_norm": 4.707404613494873,
+      "kl": 2.776157133281231,
+      "learning_rate": 9.511901296515576e-07,
+      "loss": 0.111,
+      "num_tokens": 6508160.0,
+      "reward": 0.76953125,
+      "reward_std": 0.009979372844099998,
+      "rewards//mean": 0.76953125,
+      "rewards//std": 0.028493236750364304,
+      "step": 753
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1508,
+      "grad_norm": 5.177443981170654,
+      "kl": 1.1029942650347948,
+      "learning_rate": 9.510532872691623e-07,
+      "loss": 0.0441,
+      "num_tokens": 6516832.0,
+      "reward": 0.73565673828125,
+      "reward_std": 0.008477877825498581,
+      "rewards//mean": 0.73565673828125,
+      "rewards//std": 0.02576165273785591,
+      "step": 754
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.151,
+      "grad_norm": 5.199323654174805,
+      "kl": 1.3697152510285378,
+      "learning_rate": 9.509162632025569e-07,
+      "loss": 0.0548,
+      "num_tokens": 6525520.0,
+      "reward": 0.7672119140625,
+      "reward_std": 0.007089829538017511,
+      "rewards//mean": 0.7672119140625,
+      "rewards//std": 0.0234345942735672,
+      "step": 755
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1512,
+      "grad_norm": 10.65485668182373,
+      "kl": 6.130337009206414,
+      "learning_rate": 9.507790575069345e-07,
+      "loss": 0.2452,
+      "num_tokens": 6534128.0,
+      "reward": 0.75823974609375,
+      "reward_std": 0.015302046202123165,
+      "rewards//mean": 0.75823974609375,
+      "rewards//std": 0.030154384672641754,
+      "step": 756
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1514,
+      "grad_norm": 6.569500923156738,
+      "kl": 0.9125564694404602,
+      "learning_rate": 9.506416702375617e-07,
+      "loss": 0.0365,
+      "num_tokens": 6542736.0,
+      "reward": 0.73785400390625,
+      "reward_std": 0.005343678407371044,
+      "rewards//mean": 0.73785400390625,
+      "rewards//std": 0.026983682066202164,
+      "step": 757
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1516,
+      "grad_norm": 4.671652317047119,
+      "kl": 1.829518985003233,
+      "learning_rate": 9.505041014497779e-07,
+      "loss": 0.0732,
+      "num_tokens": 6551352.0,
+      "reward": 0.75714111328125,
+      "reward_std": 0.007469784002751112,
+      "rewards//mean": 0.75714111328125,
+      "rewards//std": 0.02302626520395279,
+      "step": 758
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1518,
+      "grad_norm": 5.878870487213135,
+      "kl": 4.019355835393071,
+      "learning_rate": 9.503663511989962e-07,
+      "loss": 0.1608,
+      "num_tokens": 6560040.0,
+      "reward": 0.7625732421875,
+      "reward_std": 0.010441871359944344,
+      "rewards//mean": 0.7625732421875,
+      "rewards//std": 0.02873421274125576,
+      "step": 759
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.152,
+      "grad_norm": 4.517607688903809,
+      "kl": 3.985830817371607,
+      "learning_rate": 9.502284195407018e-07,
+      "loss": 0.1594,
+      "num_tokens": 6568704.0,
+      "reward": 0.73724365234375,
+      "reward_std": 0.012059698812663555,
+      "rewards//mean": 0.73724365234375,
+      "rewards//std": 0.03146043419837952,
+      "step": 760
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1522,
+      "grad_norm": 10.41141128540039,
+      "kl": 4.386840686202049,
+      "learning_rate": 9.500903065304539e-07,
+      "loss": 0.1755,
+      "num_tokens": 6577336.0,
+      "reward": 0.74603271484375,
+      "reward_std": 0.009094847366213799,
+      "rewards//mean": 0.74603271484375,
+      "rewards//std": 0.025272227823734283,
+      "step": 761
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1524,
+      "grad_norm": 5.01835298538208,
+      "kl": 3.469432517886162,
+      "learning_rate": 9.499520122238845e-07,
+      "loss": 0.1388,
+      "num_tokens": 6585944.0,
+      "reward": 0.7529296875,
+      "reward_std": 0.00730510801076889,
+      "rewards//mean": 0.7529296875,
+      "rewards//std": 0.03261526674032211,
+      "step": 762
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1526,
+      "grad_norm": 4.169670104980469,
+      "kl": 2.461470915004611,
+      "learning_rate": 9.498135366766982e-07,
+      "loss": 0.0985,
+      "num_tokens": 6594568.0,
+      "reward": 0.75091552734375,
+      "reward_std": 0.0068618846125900745,
+      "rewards//mean": 0.75091552734375,
+      "rewards//std": 0.02700723148882389,
+      "step": 763
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1528,
+      "grad_norm": 4.55186128616333,
+      "kl": 1.9192438144236803,
+      "learning_rate": 9.496748799446732e-07,
+      "loss": 0.0768,
+      "num_tokens": 6603064.0,
+      "reward": 0.74798583984375,
+      "reward_std": 0.010374845936894417,
+      "rewards//mean": 0.74798583984375,
+      "rewards//std": 0.025798644870519638,
+      "step": 764
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.153,
+      "grad_norm": 5.042366981506348,
+      "kl": 2.082818355411291,
+      "learning_rate": 9.495360420836602e-07,
+      "loss": 0.0833,
+      "num_tokens": 6611760.0,
+      "reward": 0.75909423828125,
+      "reward_std": 0.010806964710354805,
+      "rewards//mean": 0.75909423828125,
+      "rewards//std": 0.027081677690148354,
+      "step": 765
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1532,
+      "grad_norm": 4.896661758422852,
+      "kl": 3.562807273119688,
+      "learning_rate": 9.493970231495834e-07,
+      "loss": 0.1425,
+      "num_tokens": 6620544.0,
+      "reward": 0.784912109375,
+      "reward_std": 0.010713660158216953,
+      "rewards//mean": 0.784912109375,
+      "rewards//std": 0.034392718225717545,
+      "step": 766
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1534,
+      "grad_norm": 6.086737155914307,
+      "kl": 3.450151313096285,
+      "learning_rate": 9.492578231984393e-07,
+      "loss": 0.138,
+      "num_tokens": 6629192.0,
+      "reward": 0.7493896484375,
+      "reward_std": 0.009554892778396606,
+      "rewards//mean": 0.7493896484375,
+      "rewards//std": 0.029250076040625572,
+      "step": 767
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1536,
+      "grad_norm": 9.301119804382324,
+      "kl": 4.979688977822661,
+      "learning_rate": 9.491184422862979e-07,
+      "loss": 0.1992,
+      "num_tokens": 6637832.0,
+      "reward": 0.73736572265625,
+      "reward_std": 0.01189520675688982,
+      "rewards//mean": 0.73736572265625,
+      "rewards//std": 0.030771559104323387,
+      "step": 768
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1538,
+      "grad_norm": 6.5520405769348145,
+      "kl": 3.4958593603223562,
+      "learning_rate": 9.489788804693015e-07,
+      "loss": 0.1398,
+      "num_tokens": 6646552.0,
+      "reward": 0.7847900390625,
+      "reward_std": 0.015342256985604763,
+      "rewards//mean": 0.7847900390625,
+      "rewards//std": 0.030415115877985954,
+      "step": 769
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.154,
+      "grad_norm": 4.01801061630249,
+      "kl": 1.6592401564121246,
+      "learning_rate": 9.488391378036659e-07,
+      "loss": 0.0664,
+      "num_tokens": 6655176.0,
+      "reward": 0.78173828125,
+      "reward_std": 0.007254729513078928,
+      "rewards//mean": 0.78173828125,
+      "rewards//std": 0.018394500017166138,
+      "step": 770
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1542,
+      "grad_norm": 5.3736371994018555,
+      "kl": 2.364370469003916,
+      "learning_rate": 9.486992143456791e-07,
+      "loss": 0.0946,
+      "num_tokens": 6663840.0,
+      "reward": 0.76715087890625,
+      "reward_std": 0.008144975639879704,
+      "rewards//mean": 0.76715087890625,
+      "rewards//std": 0.030509719625115395,
+      "step": 771
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1544,
+      "grad_norm": 4.031811714172363,
+      "kl": 2.198170654475689,
+      "learning_rate": 9.485591101517026e-07,
+      "loss": 0.0879,
+      "num_tokens": 6672440.0,
+      "reward": 0.7484130859375,
+      "reward_std": 0.008436856791377068,
+      "rewards//mean": 0.7484130859375,
+      "rewards//std": 0.021751515567302704,
+      "step": 772
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1546,
+      "grad_norm": 3.4090960025787354,
+      "kl": 0.6820312235504389,
+      "learning_rate": 9.4841882527817e-07,
+      "loss": 0.0273,
+      "num_tokens": 6680960.0,
+      "reward": 0.74945068359375,
+      "reward_std": 0.007616454269737005,
+      "rewards//mean": 0.74945068359375,
+      "rewards//std": 0.023113535717129707,
+      "step": 773
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1548,
+      "grad_norm": 4.615508079528809,
+      "kl": 1.7933346442878246,
+      "learning_rate": 9.482783597815882e-07,
+      "loss": 0.0717,
+      "num_tokens": 6689760.0,
+      "reward": 0.77838134765625,
+      "reward_std": 0.009177163243293762,
+      "rewards//mean": 0.77838134765625,
+      "rewards//std": 0.028075717389583588,
+      "step": 774
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.155,
+      "grad_norm": 5.0275068283081055,
+      "kl": 2.915667874738574,
+      "learning_rate": 9.481377137185369e-07,
+      "loss": 0.1166,
+      "num_tokens": 6698392.0,
+      "reward": 0.785400390625,
+      "reward_std": 0.011071623302996159,
+      "rewards//mean": 0.785400390625,
+      "rewards//std": 0.02442800998687744,
+      "step": 775
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1552,
+      "grad_norm": 4.190218925476074,
+      "kl": 2.2015086244791746,
+      "learning_rate": 9.479968871456679e-07,
+      "loss": 0.0881,
+      "num_tokens": 6707040.0,
+      "reward": 0.7630615234375,
+      "reward_std": 0.007179578300565481,
+      "rewards//mean": 0.7630615234375,
+      "rewards//std": 0.026376372203230858,
+      "step": 776
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1554,
+      "grad_norm": 5.644638538360596,
+      "kl": 1.7162752319127321,
+      "learning_rate": 9.478558801197064e-07,
+      "loss": 0.0687,
+      "num_tokens": 6715672.0,
+      "reward": 0.7744140625,
+      "reward_std": 0.007982452400028706,
+      "rewards//mean": 0.7744140625,
+      "rewards//std": 0.019722526893019676,
+      "step": 777
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1556,
+      "grad_norm": 4.0156941413879395,
+      "kl": 2.7852271664887667,
+      "learning_rate": 9.4771469269745e-07,
+      "loss": 0.1114,
+      "num_tokens": 6724312.0,
+      "reward": 0.7691650390625,
+      "reward_std": 0.010146573185920715,
+      "rewards//mean": 0.7691650390625,
+      "rewards//std": 0.027671460062265396,
+      "step": 778
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1558,
+      "grad_norm": 7.933249473571777,
+      "kl": 4.763269757851958,
+      "learning_rate": 9.475733249357688e-07,
+      "loss": 0.1905,
+      "num_tokens": 6732968.0,
+      "reward": 0.77203369140625,
+      "reward_std": 0.013013869524002075,
+      "rewards//mean": 0.77203369140625,
+      "rewards//std": 0.027981745079159737,
+      "step": 779
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.156,
+      "grad_norm": 12.175159454345703,
+      "kl": 5.108350550755858,
+      "learning_rate": 9.474317768916059e-07,
+      "loss": 0.2043,
+      "num_tokens": 6741616.0,
+      "reward": 0.76422119140625,
+      "reward_std": 0.009025199338793755,
+      "rewards//mean": 0.76422119140625,
+      "rewards//std": 0.022524451836943626,
+      "step": 780
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1562,
+      "grad_norm": 9.010479927062988,
+      "kl": 2.115995079278946,
+      "learning_rate": 9.472900486219768e-07,
+      "loss": 0.0846,
+      "num_tokens": 6750248.0,
+      "reward": 0.77886962890625,
+      "reward_std": 0.011248920112848282,
+      "rewards//mean": 0.77886962890625,
+      "rewards//std": 0.030533524230122566,
+      "step": 781
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1564,
+      "grad_norm": 5.800589561462402,
+      "kl": 2.6221635565161705,
+      "learning_rate": 9.471481401839696e-07,
+      "loss": 0.1049,
+      "num_tokens": 6758784.0,
+      "reward": 0.7642822265625,
+      "reward_std": 0.011129805818200111,
+      "rewards//mean": 0.7642822265625,
+      "rewards//std": 0.02669582888484001,
+      "step": 782
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1566,
+      "grad_norm": 4.211091995239258,
+      "kl": 4.653603246435523,
+      "learning_rate": 9.470060516347449e-07,
+      "loss": 0.1861,
+      "num_tokens": 6767448.0,
+      "reward": 0.78985595703125,
+      "reward_std": 0.014145842753350735,
+      "rewards//mean": 0.78985595703125,
+      "rewards//std": 0.020530959591269493,
+      "step": 783
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1568,
+      "grad_norm": 4.587401866912842,
+      "kl": 3.3550094813108444,
+      "learning_rate": 9.468637830315362e-07,
+      "loss": 0.1342,
+      "num_tokens": 6776224.0,
+      "reward": 0.7330322265625,
+      "reward_std": 0.011062645353376865,
+      "rewards//mean": 0.7330322265625,
+      "rewards//std": 0.04221653565764427,
+      "step": 784
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.157,
+      "grad_norm": 5.721545696258545,
+      "kl": 3.0369136724621058,
+      "learning_rate": 9.467213344316491e-07,
+      "loss": 0.1215,
+      "num_tokens": 6784832.0,
+      "reward": 0.7496337890625,
+      "reward_std": 0.008115509524941444,
+      "rewards//mean": 0.7496337890625,
+      "rewards//std": 0.03495047613978386,
+      "step": 785
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1572,
+      "grad_norm": 6.887928009033203,
+      "kl": 3.6168970745056868,
+      "learning_rate": 9.465787058924619e-07,
+      "loss": 0.1447,
+      "num_tokens": 6793464.0,
+      "reward": 0.78460693359375,
+      "reward_std": 0.006885099224746227,
+      "rewards//mean": 0.78460693359375,
+      "rewards//std": 0.02213943377137184,
+      "step": 786
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1574,
+      "grad_norm": 4.287591457366943,
+      "kl": 3.2999197766184807,
+      "learning_rate": 9.464358974714252e-07,
+      "loss": 0.132,
+      "num_tokens": 6801992.0,
+      "reward": 0.77069091796875,
+      "reward_std": 0.013432151637971401,
+      "rewards//mean": 0.77069091796875,
+      "rewards//std": 0.029137752950191498,
+      "step": 787
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1576,
+      "grad_norm": 6.390154838562012,
+      "kl": 2.886853411793709,
+      "learning_rate": 9.462929092260628e-07,
+      "loss": 0.1155,
+      "num_tokens": 6810688.0,
+      "reward": 0.74188232421875,
+      "reward_std": 0.013014253228902817,
+      "rewards//mean": 0.74188232421875,
+      "rewards//std": 0.03236407786607742,
+      "step": 788
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1578,
+      "grad_norm": 7.804890155792236,
+      "kl": 5.38600605353713,
+      "learning_rate": 9.461497412139696e-07,
+      "loss": 0.2154,
+      "num_tokens": 6819336.0,
+      "reward": 0.75628662109375,
+      "reward_std": 0.014761541038751602,
+      "rewards//mean": 0.75628662109375,
+      "rewards//std": 0.03839721903204918,
+      "step": 789
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.158,
+      "grad_norm": 11.392706871032715,
+      "kl": 5.6631733775138855,
+      "learning_rate": 9.460063934928141e-07,
+      "loss": 0.2265,
+      "num_tokens": 6828088.0,
+      "reward": 0.7474365234375,
+      "reward_std": 0.011669349856674671,
+      "rewards//mean": 0.7474365234375,
+      "rewards//std": 0.028933709487318993,
+      "step": 790
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1582,
+      "grad_norm": 6.763251304626465,
+      "kl": 4.461639614775777,
+      "learning_rate": 9.458628661203366e-07,
+      "loss": 0.1785,
+      "num_tokens": 6836808.0,
+      "reward": 0.7615966796875,
+      "reward_std": 0.010639270767569542,
+      "rewards//mean": 0.7615966796875,
+      "rewards//std": 0.014894389547407627,
+      "step": 791
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1584,
+      "grad_norm": 4.676641941070557,
+      "kl": 3.748870886862278,
+      "learning_rate": 9.4571915915435e-07,
+      "loss": 0.15,
+      "num_tokens": 6845400.0,
+      "reward": 0.75372314453125,
+      "reward_std": 0.007258172146975994,
+      "rewards//mean": 0.75372314453125,
+      "rewards//std": 0.017859095707535744,
+      "step": 792
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1586,
+      "grad_norm": 4.426313877105713,
+      "kl": 3.1063752956688404,
+      "learning_rate": 9.455752726527392e-07,
+      "loss": 0.1243,
+      "num_tokens": 6854096.0,
+      "reward": 0.7286376953125,
+      "reward_std": 0.011803104542195797,
+      "rewards//mean": 0.7286376953125,
+      "rewards//std": 0.030851947143673897,
+      "step": 793
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1588,
+      "grad_norm": 3.786665916442871,
+      "kl": 1.685317488387227,
+      "learning_rate": 9.454312066734622e-07,
+      "loss": 0.0674,
+      "num_tokens": 6862656.0,
+      "reward": 0.72662353515625,
+      "reward_std": 0.0077147455886006355,
+      "rewards//mean": 0.72662353515625,
+      "rewards//std": 0.020392615348100662,
+      "step": 794
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.159,
+      "grad_norm": 4.603367328643799,
+      "kl": 1.2162437960505486,
+      "learning_rate": 9.452869612745483e-07,
+      "loss": 0.0486,
+      "num_tokens": 6871248.0,
+      "reward": 0.7705078125,
+      "reward_std": 0.008626492694020271,
+      "rewards//mean": 0.7705078125,
+      "rewards//std": 0.021687744185328484,
+      "step": 795
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1592,
+      "grad_norm": 4.271348476409912,
+      "kl": 1.668197050690651,
+      "learning_rate": 9.451425365140994e-07,
+      "loss": 0.0667,
+      "num_tokens": 6879968.0,
+      "reward": 0.74993896484375,
+      "reward_std": 0.006746210157871246,
+      "rewards//mean": 0.74993896484375,
+      "rewards//std": 0.030317597091197968,
+      "step": 796
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1594,
+      "grad_norm": 6.6405510902404785,
+      "kl": 3.7507582679390907,
+      "learning_rate": 9.449979324502903e-07,
+      "loss": 0.15,
+      "num_tokens": 6888536.0,
+      "reward": 0.77099609375,
+      "reward_std": 0.01573047786951065,
+      "rewards//mean": 0.77099609375,
+      "rewards//std": 0.024084772914648056,
+      "step": 797
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1596,
+      "grad_norm": 4.867390155792236,
+      "kl": 2.5980178639292717,
+      "learning_rate": 9.448531491413672e-07,
+      "loss": 0.1039,
+      "num_tokens": 6897080.0,
+      "reward": 0.77197265625,
+      "reward_std": 0.01075395755469799,
+      "rewards//mean": 0.77197265625,
+      "rewards//std": 0.02174908109009266,
+      "step": 798
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1598,
+      "grad_norm": 3.9256911277770996,
+      "kl": 0.747510327026248,
+      "learning_rate": 9.447081866456487e-07,
+      "loss": 0.0299,
+      "num_tokens": 6905712.0,
+      "reward": 0.75689697265625,
+      "reward_std": 0.0040665497072041035,
+      "rewards//mean": 0.75689697265625,
+      "rewards//std": 0.026224639266729355,
+      "step": 799
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.16,
+      "grad_norm": 8.939791679382324,
+      "kl": 2.05123102478683,
+      "learning_rate": 9.445630450215259e-07,
+      "loss": 0.082,
+      "num_tokens": 6914304.0,
+      "reward": 0.7764892578125,
+      "reward_std": 0.009935218840837479,
+      "rewards//mean": 0.7764892578125,
+      "rewards//std": 0.023566000163555145,
+      "step": 800
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1602,
+      "grad_norm": 7.597315311431885,
+      "kl": 1.4521824270486832,
+      "learning_rate": 9.444177243274617e-07,
+      "loss": 0.0581,
+      "num_tokens": 6922960.0,
+      "reward": 0.7889404296875,
+      "reward_std": 0.007975172251462936,
+      "rewards//mean": 0.7889404296875,
+      "rewards//std": 0.02867092378437519,
+      "step": 801
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1604,
+      "grad_norm": 4.933373928070068,
+      "kl": 1.937702227383852,
+      "learning_rate": 9.442722246219913e-07,
+      "loss": 0.0775,
+      "num_tokens": 6931632.0,
+      "reward": 0.73895263671875,
+      "reward_std": 0.006113705225288868,
+      "rewards//mean": 0.73895263671875,
+      "rewards//std": 0.03296652063727379,
+      "step": 802
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1606,
+      "grad_norm": 4.05463981628418,
+      "kl": 2.27868514880538,
+      "learning_rate": 9.441265459637219e-07,
+      "loss": 0.0911,
+      "num_tokens": 6940272.0,
+      "reward": 0.77081298828125,
+      "reward_std": 0.00799493957310915,
+      "rewards//mean": 0.77081298828125,
+      "rewards//std": 0.025553978979587555,
+      "step": 803
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1608,
+      "grad_norm": 5.101175785064697,
+      "kl": 4.256691545248032,
+      "learning_rate": 9.43980688411333e-07,
+      "loss": 0.1703,
+      "num_tokens": 6948856.0,
+      "reward": 0.74481201171875,
+      "reward_std": 0.019318178296089172,
+      "rewards//mean": 0.74481201171875,
+      "rewards//std": 0.03434465453028679,
+      "step": 804
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.161,
+      "grad_norm": 5.4669952392578125,
+      "kl": 2.1037301030009985,
+      "learning_rate": 9.438346520235758e-07,
+      "loss": 0.0841,
+      "num_tokens": 6957592.0,
+      "reward": 0.75518798828125,
+      "reward_std": 0.008407503366470337,
+      "rewards//mean": 0.75518798828125,
+      "rewards//std": 0.023105019703507423,
+      "step": 805
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1612,
+      "grad_norm": 5.856110572814941,
+      "kl": 3.3386458940804005,
+      "learning_rate": 9.436884368592739e-07,
+      "loss": 0.1335,
+      "num_tokens": 6966240.0,
+      "reward": 0.775390625,
+      "reward_std": 0.010535828769207,
+      "rewards//mean": 0.775390625,
+      "rewards//std": 0.02530582621693611,
+      "step": 806
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1614,
+      "grad_norm": 5.265435218811035,
+      "kl": 2.4264450185000896,
+      "learning_rate": 9.435420429773227e-07,
+      "loss": 0.0971,
+      "num_tokens": 6974904.0,
+      "reward": 0.7279052734375,
+      "reward_std": 0.01252413634210825,
+      "rewards//mean": 0.7279052734375,
+      "rewards//std": 0.03498857095837593,
+      "step": 807
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1616,
+      "grad_norm": 5.059149265289307,
+      "kl": 1.7861047461628914,
+      "learning_rate": 9.433954704366896e-07,
+      "loss": 0.0714,
+      "num_tokens": 6983504.0,
+      "reward": 0.77508544921875,
+      "reward_std": 0.007414050400257111,
+      "rewards//mean": 0.77508544921875,
+      "rewards//std": 0.023144295439124107,
+      "step": 808
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1618,
+      "grad_norm": 4.534353733062744,
+      "kl": 3.3108605053275824,
+      "learning_rate": 9.43248719296414e-07,
+      "loss": 0.1324,
+      "num_tokens": 6992216.0,
+      "reward": 0.756591796875,
+      "reward_std": 0.011271260678768158,
+      "rewards//mean": 0.756591796875,
+      "rewards//std": 0.02740016020834446,
+      "step": 809
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.162,
+      "grad_norm": 5.283889293670654,
+      "kl": 3.1531424652785063,
+      "learning_rate": 9.431017896156073e-07,
+      "loss": 0.1261,
+      "num_tokens": 7000816.0,
+      "reward": 0.7689208984375,
+      "reward_std": 0.015889810398221016,
+      "rewards//mean": 0.7689208984375,
+      "rewards//std": 0.03676409646868706,
+      "step": 810
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1622,
+      "grad_norm": 5.881231307983398,
+      "kl": 1.6919131707400084,
+      "learning_rate": 9.429546814534528e-07,
+      "loss": 0.0677,
+      "num_tokens": 7009480.0,
+      "reward": 0.7716064453125,
+      "reward_std": 0.005492859985679388,
+      "rewards//mean": 0.7716064453125,
+      "rewards//std": 0.020552054047584534,
+      "step": 811
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1624,
+      "grad_norm": 9.305164337158203,
+      "kl": 4.946976518258452,
+      "learning_rate": 9.428073948692054e-07,
+      "loss": 0.1979,
+      "num_tokens": 7018136.0,
+      "reward": 0.73931884765625,
+      "reward_std": 0.01171068660914898,
+      "rewards//mean": 0.73931884765625,
+      "rewards//std": 0.02789396606385708,
+      "step": 812
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1626,
+      "grad_norm": 5.397275447845459,
+      "kl": 4.610825087875128,
+      "learning_rate": 9.426599299221924e-07,
+      "loss": 0.1844,
+      "num_tokens": 7026824.0,
+      "reward": 0.75689697265625,
+      "reward_std": 0.016051501035690308,
+      "rewards//mean": 0.75689697265625,
+      "rewards//std": 0.03400936350226402,
+      "step": 813
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1628,
+      "grad_norm": 6.010089874267578,
+      "kl": 3.0963895712047815,
+      "learning_rate": 9.425122866718127e-07,
+      "loss": 0.1239,
+      "num_tokens": 7035488.0,
+      "reward": 0.74285888671875,
+      "reward_std": 0.009649118408560753,
+      "rewards//mean": 0.74285888671875,
+      "rewards//std": 0.02659890055656433,
+      "step": 814
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.163,
+      "grad_norm": 5.974343776702881,
+      "kl": 2.298018941655755,
+      "learning_rate": 9.423644651775368e-07,
+      "loss": 0.0919,
+      "num_tokens": 7044072.0,
+      "reward": 0.7584228515625,
+      "reward_std": 0.013415565714240074,
+      "rewards//mean": 0.7584228515625,
+      "rewards//std": 0.023501677438616753,
+      "step": 815
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1632,
+      "grad_norm": 10.788091659545898,
+      "kl": 4.28102938272059,
+      "learning_rate": 9.422164654989071e-07,
+      "loss": 0.1712,
+      "num_tokens": 7052744.0,
+      "reward": 0.76611328125,
+      "reward_std": 0.011416466906666756,
+      "rewards//mean": 0.76611328125,
+      "rewards//std": 0.027370311319828033,
+      "step": 816
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1634,
+      "grad_norm": 5.269587516784668,
+      "kl": 1.0280618537217379,
+      "learning_rate": 9.420682876955381e-07,
+      "loss": 0.0411,
+      "num_tokens": 7061384.0,
+      "reward": 0.7451171875,
+      "reward_std": 0.00575895793735981,
+      "rewards//mean": 0.7451171875,
+      "rewards//std": 0.02525792457163334,
+      "step": 817
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1636,
+      "grad_norm": 3.6746578216552734,
+      "kl": 3.659264462068677,
+      "learning_rate": 9.419199318271156e-07,
+      "loss": 0.1464,
+      "num_tokens": 7070008.0,
+      "reward": 0.747314453125,
+      "reward_std": 0.010371357202529907,
+      "rewards//mean": 0.747314453125,
+      "rewards//std": 0.028897326439619064,
+      "step": 818
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1638,
+      "grad_norm": 6.09009313583374,
+      "kl": 4.993275761604309,
+      "learning_rate": 9.417713979533974e-07,
+      "loss": 0.1997,
+      "num_tokens": 7078680.0,
+      "reward": 0.77337646484375,
+      "reward_std": 0.012450158596038818,
+      "rewards//mean": 0.77337646484375,
+      "rewards//std": 0.023514797911047935,
+      "step": 819
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.164,
+      "grad_norm": 6.171932697296143,
+      "kl": 2.3480502869933844,
+      "learning_rate": 9.41622686134213e-07,
+      "loss": 0.0939,
+      "num_tokens": 7087336.0,
+      "reward": 0.73858642578125,
+      "reward_std": 0.004920503124594688,
+      "rewards//mean": 0.73858642578125,
+      "rewards//std": 0.026958424597978592,
+      "step": 820
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1642,
+      "grad_norm": 4.886020660400391,
+      "kl": 4.0925056375563145,
+      "learning_rate": 9.414737964294634e-07,
+      "loss": 0.1637,
+      "num_tokens": 7096016.0,
+      "reward": 0.75732421875,
+      "reward_std": 0.01674531400203705,
+      "rewards//mean": 0.75732421875,
+      "rewards//std": 0.026461461558938026,
+      "step": 821
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1644,
+      "grad_norm": 12.292215347290039,
+      "kl": 5.2495632860809565,
+      "learning_rate": 9.413247288991215e-07,
+      "loss": 0.21,
+      "num_tokens": 7104736.0,
+      "reward": 0.724365234375,
+      "reward_std": 0.00853973813354969,
+      "rewards//mean": 0.724365234375,
+      "rewards//std": 0.031062457710504532,
+      "step": 822
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1646,
+      "grad_norm": 6.820827007293701,
+      "kl": 3.6908694114536047,
+      "learning_rate": 9.411754836032314e-07,
+      "loss": 0.1476,
+      "num_tokens": 7113368.0,
+      "reward": 0.757080078125,
+      "reward_std": 0.007114241365343332,
+      "rewards//mean": 0.757080078125,
+      "rewards//std": 0.025457311421632767,
+      "step": 823
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1648,
+      "grad_norm": 5.6415324211120605,
+      "kl": 1.4121773652732372,
+      "learning_rate": 9.410260606019094e-07,
+      "loss": 0.0565,
+      "num_tokens": 7122128.0,
+      "reward": 0.77734375,
+      "reward_std": 0.00641645397990942,
+      "rewards//mean": 0.77734375,
+      "rewards//std": 0.02536318637430668,
+      "step": 824
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.165,
+      "grad_norm": 4.019225120544434,
+      "kl": 1.3440431728959084,
+      "learning_rate": 9.408764599553428e-07,
+      "loss": 0.0538,
+      "num_tokens": 7130776.0,
+      "reward": 0.75555419921875,
+      "reward_std": 0.007369856350123882,
+      "rewards//mean": 0.75555419921875,
+      "rewards//std": 0.01826639473438263,
+      "step": 825
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1652,
+      "grad_norm": 4.040704727172852,
+      "kl": 1.9375916495919228,
+      "learning_rate": 9.40726681723791e-07,
+      "loss": 0.0775,
+      "num_tokens": 7139528.0,
+      "reward": 0.7393798828125,
+      "reward_std": 0.007135326508432627,
+      "rewards//mean": 0.7393798828125,
+      "rewards//std": 0.032818615436553955,
+      "step": 826
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1654,
+      "grad_norm": 4.428711891174316,
+      "kl": 2.506573861464858,
+      "learning_rate": 9.405767259675844e-07,
+      "loss": 0.1003,
+      "num_tokens": 7148128.0,
+      "reward": 0.72418212890625,
+      "reward_std": 0.009128805249929428,
+      "rewards//mean": 0.72418212890625,
+      "rewards//std": 0.046274978667497635,
+      "step": 827
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1656,
+      "grad_norm": 6.2188568115234375,
+      "kl": 1.8739633075892925,
+      "learning_rate": 9.404265927471253e-07,
+      "loss": 0.075,
+      "num_tokens": 7156728.0,
+      "reward": 0.7354736328125,
+      "reward_std": 0.008282147347927094,
+      "rewards//mean": 0.7354736328125,
+      "rewards//std": 0.03260389342904091,
+      "step": 828
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1658,
+      "grad_norm": 4.146193981170654,
+      "kl": 0.5284234452992678,
+      "learning_rate": 9.402762821228874e-07,
+      "loss": 0.0211,
+      "num_tokens": 7165408.0,
+      "reward": 0.76214599609375,
+      "reward_std": 0.005288444459438324,
+      "rewards//mean": 0.76214599609375,
+      "rewards//std": 0.02323112264275551,
+      "step": 829
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.166,
+      "grad_norm": 3.5680792331695557,
+      "kl": 2.4490291960537434,
+      "learning_rate": 9.401257941554156e-07,
+      "loss": 0.098,
+      "num_tokens": 7174016.0,
+      "reward": 0.76226806640625,
+      "reward_std": 0.009816572070121765,
+      "rewards//mean": 0.76226806640625,
+      "rewards//std": 0.03335590288043022,
+      "step": 830
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1662,
+      "grad_norm": 4.464192867279053,
+      "kl": 0.9390498735010624,
+      "learning_rate": 9.399751289053266e-07,
+      "loss": 0.0376,
+      "num_tokens": 7182656.0,
+      "reward": 0.76025390625,
+      "reward_std": 0.0055123730562627316,
+      "rewards//mean": 0.76025390625,
+      "rewards//std": 0.021389752626419067,
+      "step": 831
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1664,
+      "grad_norm": 3.8422210216522217,
+      "kl": 1.9241707921028137,
+      "learning_rate": 9.398242864333083e-07,
+      "loss": 0.077,
+      "num_tokens": 7191360.0,
+      "reward": 0.76580810546875,
+      "reward_std": 0.0072711799293756485,
+      "rewards//mean": 0.76580810546875,
+      "rewards//std": 0.03402137756347656,
+      "step": 832
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1666,
+      "grad_norm": 5.312897205352783,
+      "kl": 3.2794720102101564,
+      "learning_rate": 9.396732668001199e-07,
+      "loss": 0.1312,
+      "num_tokens": 7200080.0,
+      "reward": 0.7540283203125,
+      "reward_std": 0.007833273150026798,
+      "rewards//mean": 0.7540283203125,
+      "rewards//std": 0.01580943912267685,
+      "step": 833
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1668,
+      "grad_norm": 3.0219478607177734,
+      "kl": 0.4920041970908642,
+      "learning_rate": 9.395220700665922e-07,
+      "loss": 0.0197,
+      "num_tokens": 7208816.0,
+      "reward": 0.7708740234375,
+      "reward_std": 0.004750157240778208,
+      "rewards//mean": 0.7708740234375,
+      "rewards//std": 0.026108719408512115,
+      "step": 834
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.167,
+      "grad_norm": 6.649507522583008,
+      "kl": 2.959937596693635,
+      "learning_rate": 9.393706962936274e-07,
+      "loss": 0.1184,
+      "num_tokens": 7217488.0,
+      "reward": 0.76373291015625,
+      "reward_std": 0.008829772472381592,
+      "rewards//mean": 0.76373291015625,
+      "rewards//std": 0.029438046738505363,
+      "step": 835
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1672,
+      "grad_norm": 3.517529010772705,
+      "kl": 0.6590344868600368,
+      "learning_rate": 9.392191455421987e-07,
+      "loss": 0.0264,
+      "num_tokens": 7226024.0,
+      "reward": 0.76226806640625,
+      "reward_std": 0.004965063184499741,
+      "rewards//mean": 0.76226806640625,
+      "rewards//std": 0.022465234622359276,
+      "step": 836
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1674,
+      "grad_norm": 4.816415309906006,
+      "kl": 2.957753771916032,
+      "learning_rate": 9.390674178733507e-07,
+      "loss": 0.1183,
+      "num_tokens": 7234672.0,
+      "reward": 0.75238037109375,
+      "reward_std": 0.008637012913823128,
+      "rewards//mean": 0.75238037109375,
+      "rewards//std": 0.030326582491397858,
+      "step": 837
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1676,
+      "grad_norm": 7.869420528411865,
+      "kl": 4.866269508376718,
+      "learning_rate": 9.389155133481992e-07,
+      "loss": 0.1947,
+      "num_tokens": 7243248.0,
+      "reward": 0.73876953125,
+      "reward_std": 0.009045137092471123,
+      "rewards//mean": 0.73876953125,
+      "rewards//std": 0.030556384474039078,
+      "step": 838
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1678,
+      "grad_norm": 4.110578536987305,
+      "kl": 1.7108943182975054,
+      "learning_rate": 9.387634320279314e-07,
+      "loss": 0.0684,
+      "num_tokens": 7252024.0,
+      "reward": 0.744384765625,
+      "reward_std": 0.006312161218374968,
+      "rewards//mean": 0.744384765625,
+      "rewards//std": 0.026244329288601875,
+      "step": 839
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.168,
+      "grad_norm": 5.340089797973633,
+      "kl": 3.5931700356304646,
+      "learning_rate": 9.386111739738056e-07,
+      "loss": 0.1437,
+      "num_tokens": 7260632.0,
+      "reward": 0.75054931640625,
+      "reward_std": 0.008358034305274487,
+      "rewards//mean": 0.75054931640625,
+      "rewards//std": 0.03334864228963852,
+      "step": 840
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1682,
+      "grad_norm": 4.9973931312561035,
+      "kl": 3.812736565247178,
+      "learning_rate": 9.384587392471514e-07,
+      "loss": 0.1525,
+      "num_tokens": 7269232.0,
+      "reward": 0.7659912109375,
+      "reward_std": 0.01163836196064949,
+      "rewards//mean": 0.7659912109375,
+      "rewards//std": 0.022742237895727158,
+      "step": 841
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1684,
+      "grad_norm": 5.224663734436035,
+      "kl": 2.3228061739355326,
+      "learning_rate": 9.383061279093696e-07,
+      "loss": 0.0929,
+      "num_tokens": 7277888.0,
+      "reward": 0.76190185546875,
+      "reward_std": 0.008733944036066532,
+      "rewards//mean": 0.76190185546875,
+      "rewards//std": 0.014934353530406952,
+      "step": 842
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1686,
+      "grad_norm": 6.6156439781188965,
+      "kl": 2.879060560837388,
+      "learning_rate": 9.381533400219317e-07,
+      "loss": 0.1152,
+      "num_tokens": 7286528.0,
+      "reward": 0.77838134765625,
+      "reward_std": 0.008324716240167618,
+      "rewards//mean": 0.77838134765625,
+      "rewards//std": 0.02941541187465191,
+      "step": 843
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1688,
+      "grad_norm": 4.613117694854736,
+      "kl": 2.078267527744174,
+      "learning_rate": 9.38000375646381e-07,
+      "loss": 0.0831,
+      "num_tokens": 7295168.0,
+      "reward": 0.7542724609375,
+      "reward_std": 0.008011859841644764,
+      "rewards//mean": 0.7542724609375,
+      "rewards//std": 0.020312009379267693,
+      "step": 844
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.169,
+      "grad_norm": 5.4162797927856445,
+      "kl": 2.765382260084152,
+      "learning_rate": 9.378472348443314e-07,
+      "loss": 0.1106,
+      "num_tokens": 7303800.0,
+      "reward": 0.74188232421875,
+      "reward_std": 0.006411589216440916,
+      "rewards//mean": 0.74188232421875,
+      "rewards//std": 0.021545499563217163,
+      "step": 845
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1692,
+      "grad_norm": 4.39764928817749,
+      "kl": 4.487566510215402,
+      "learning_rate": 9.376939176774677e-07,
+      "loss": 0.1795,
+      "num_tokens": 7312504.0,
+      "reward": 0.77227783203125,
+      "reward_std": 0.013200690969824791,
+      "rewards//mean": 0.77227783203125,
+      "rewards//std": 0.027863016352057457,
+      "step": 846
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1694,
+      "grad_norm": 12.082962989807129,
+      "kl": 4.153339693322778,
+      "learning_rate": 9.375404242075466e-07,
+      "loss": 0.1661,
+      "num_tokens": 7321176.0,
+      "reward": 0.74102783203125,
+      "reward_std": 0.005838182754814625,
+      "rewards//mean": 0.74102783203125,
+      "rewards//std": 0.02251705899834633,
+      "step": 847
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1696,
+      "grad_norm": 4.346368312835693,
+      "kl": 2.3379539754241705,
+      "learning_rate": 9.373867544963948e-07,
+      "loss": 0.0935,
+      "num_tokens": 7329792.0,
+      "reward": 0.776123046875,
+      "reward_std": 0.009924739599227905,
+      "rewards//mean": 0.776123046875,
+      "rewards//std": 0.024348560720682144,
+      "step": 848
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1698,
+      "grad_norm": 4.499940872192383,
+      "kl": 2.722176406532526,
+      "learning_rate": 9.372329086059107e-07,
+      "loss": 0.1089,
+      "num_tokens": 7338440.0,
+      "reward": 0.77099609375,
+      "reward_std": 0.007846756838262081,
+      "rewards//mean": 0.77099609375,
+      "rewards//std": 0.021882306784391403,
+      "step": 849
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.17,
+      "grad_norm": 4.038485527038574,
+      "kl": 1.8677042871713638,
+      "learning_rate": 9.370788865980632e-07,
+      "loss": 0.0747,
+      "num_tokens": 7347016.0,
+      "reward": 0.76861572265625,
+      "reward_std": 0.008194370195269585,
+      "rewards//mean": 0.76861572265625,
+      "rewards//std": 0.025379817932844162,
+      "step": 850
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1702,
+      "grad_norm": 5.125847339630127,
+      "kl": 1.8140365444123745,
+      "learning_rate": 9.369246885348925e-07,
+      "loss": 0.0726,
+      "num_tokens": 7355664.0,
+      "reward": 0.7032470703125,
+      "reward_std": 0.006922622211277485,
+      "rewards//mean": 0.7032470703125,
+      "rewards//std": 0.03520766645669937,
+      "step": 851
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1704,
+      "grad_norm": 4.963730335235596,
+      "kl": 3.619083819910884,
+      "learning_rate": 9.367703144785095e-07,
+      "loss": 0.1448,
+      "num_tokens": 7364392.0,
+      "reward": 0.76422119140625,
+      "reward_std": 0.010131246410310268,
+      "rewards//mean": 0.76422119140625,
+      "rewards//std": 0.0317067876458168,
+      "step": 852
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1706,
+      "grad_norm": 4.636470794677734,
+      "kl": 3.3408317267894745,
+      "learning_rate": 9.366157644910959e-07,
+      "loss": 0.1336,
+      "num_tokens": 7373032.0,
+      "reward": 0.76007080078125,
+      "reward_std": 0.010227615013718605,
+      "rewards//mean": 0.76007080078125,
+      "rewards//std": 0.030487382784485817,
+      "step": 853
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1708,
+      "grad_norm": 4.1317219734191895,
+      "kl": 0.7655187211930752,
+      "learning_rate": 9.364610386349047e-07,
+      "loss": 0.0306,
+      "num_tokens": 7381688.0,
+      "reward": 0.76336669921875,
+      "reward_std": 0.005188517272472382,
+      "rewards//mean": 0.76336669921875,
+      "rewards//std": 0.017850618809461594,
+      "step": 854
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.171,
+      "grad_norm": 6.155958652496338,
+      "kl": 4.210917366668582,
+      "learning_rate": 9.363061369722594e-07,
+      "loss": 0.1684,
+      "num_tokens": 7390344.0,
+      "reward": 0.7872314453125,
+      "reward_std": 0.015684831887483597,
+      "rewards//mean": 0.7872314453125,
+      "rewards//std": 0.027795907109975815,
+      "step": 855
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1712,
+      "grad_norm": 3.3234474658966064,
+      "kl": 0.5247700996696949,
+      "learning_rate": 9.361510595655544e-07,
+      "loss": 0.021,
+      "num_tokens": 7398960.0,
+      "reward": 0.7786865234375,
+      "reward_std": 0.004236909560859203,
+      "rewards//mean": 0.7786865234375,
+      "rewards//std": 0.02105558104813099,
+      "step": 856
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1714,
+      "grad_norm": 7.604945182800293,
+      "kl": 4.309569004923105,
+      "learning_rate": 9.359958064772546e-07,
+      "loss": 0.1724,
+      "num_tokens": 7407632.0,
+      "reward": 0.73480224609375,
+      "reward_std": 0.007704409770667553,
+      "rewards//mean": 0.73480224609375,
+      "rewards//std": 0.027944933623075485,
+      "step": 857
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1716,
+      "grad_norm": 9.44655990600586,
+      "kl": 2.2118360735476017,
+      "learning_rate": 9.35840377769896e-07,
+      "loss": 0.0885,
+      "num_tokens": 7416224.0,
+      "reward": 0.7835693359375,
+      "reward_std": 0.010356249287724495,
+      "rewards//mean": 0.7835693359375,
+      "rewards//std": 0.020383426919579506,
+      "step": 858
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1718,
+      "grad_norm": 4.0900163650512695,
+      "kl": 3.979547144845128,
+      "learning_rate": 9.356847735060856e-07,
+      "loss": 0.1592,
+      "num_tokens": 7424872.0,
+      "reward": 0.78057861328125,
+      "reward_std": 0.007768874987959862,
+      "rewards//mean": 0.78057861328125,
+      "rewards//std": 0.021693937480449677,
+      "step": 859
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.172,
+      "grad_norm": 5.1128830909729,
+      "kl": 2.8700168412178755,
+      "learning_rate": 9.355289937485004e-07,
+      "loss": 0.1148,
+      "num_tokens": 7433544.0,
+      "reward": 0.77447509765625,
+      "reward_std": 0.008934141136705875,
+      "rewards//mean": 0.77447509765625,
+      "rewards//std": 0.028673233464360237,
+      "step": 860
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1722,
+      "grad_norm": 4.9949870109558105,
+      "kl": 3.751866629347205,
+      "learning_rate": 9.353730385598886e-07,
+      "loss": 0.1501,
+      "num_tokens": 7442144.0,
+      "reward": 0.76397705078125,
+      "reward_std": 0.010951785370707512,
+      "rewards//mean": 0.76397705078125,
+      "rewards//std": 0.02761087566614151,
+      "step": 861
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1724,
+      "grad_norm": 5.703150272369385,
+      "kl": 3.635123133659363,
+      "learning_rate": 9.35216908003069e-07,
+      "loss": 0.1454,
+      "num_tokens": 7450776.0,
+      "reward": 0.7626953125,
+      "reward_std": 0.010348414070904255,
+      "rewards//mean": 0.7626953125,
+      "rewards//std": 0.02896534651517868,
+      "step": 862
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1726,
+      "grad_norm": 4.669232368469238,
+      "kl": 4.9712036326527596,
+      "learning_rate": 9.350606021409308e-07,
+      "loss": 0.1988,
+      "num_tokens": 7459440.0,
+      "reward": 0.78179931640625,
+      "reward_std": 0.012088283896446228,
+      "rewards//mean": 0.78179931640625,
+      "rewards//std": 0.021719740703701973,
+      "step": 863
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1728,
+      "grad_norm": 3.92879319190979,
+      "kl": 2.666841896250844,
+      "learning_rate": 9.349041210364341e-07,
+      "loss": 0.1067,
+      "num_tokens": 7468160.0,
+      "reward": 0.7828369140625,
+      "reward_std": 0.009088682942092419,
+      "rewards//mean": 0.7828369140625,
+      "rewards//std": 0.021312827244400978,
+      "step": 864
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.173,
+      "grad_norm": 4.29932975769043,
+      "kl": 2.829136600717902,
+      "learning_rate": 9.347474647526095e-07,
+      "loss": 0.1132,
+      "num_tokens": 7476752.0,
+      "reward": 0.77197265625,
+      "reward_std": 0.007671776227653027,
+      "rewards//mean": 0.77197265625,
+      "rewards//std": 0.015032478608191013,
+      "step": 865
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1732,
+      "grad_norm": 5.589138984680176,
+      "kl": 2.3940916135907173,
+      "learning_rate": 9.34590633352558e-07,
+      "loss": 0.0958,
+      "num_tokens": 7485448.0,
+      "reward": 0.77569580078125,
+      "reward_std": 0.01034168154001236,
+      "rewards//mean": 0.77569580078125,
+      "rewards//std": 0.02812366373836994,
+      "step": 866
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1734,
+      "grad_norm": 11.367834091186523,
+      "kl": 4.466622106730938,
+      "learning_rate": 9.344336268994515e-07,
+      "loss": 0.1787,
+      "num_tokens": 7494104.0,
+      "reward": 0.7293701171875,
+      "reward_std": 0.00977625884115696,
+      "rewards//mean": 0.7293701171875,
+      "rewards//std": 0.023882482200860977,
+      "step": 867
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1736,
+      "grad_norm": 7.060291290283203,
+      "kl": 3.5269526708871126,
+      "learning_rate": 9.342764454565319e-07,
+      "loss": 0.1411,
+      "num_tokens": 7502752.0,
+      "reward": 0.7628173828125,
+      "reward_std": 0.008809993974864483,
+      "rewards//mean": 0.7628173828125,
+      "rewards//std": 0.0270316731184721,
+      "step": 868
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1738,
+      "grad_norm": 4.928085803985596,
+      "kl": 2.0577304176986217,
+      "learning_rate": 9.341190890871121e-07,
+      "loss": 0.0823,
+      "num_tokens": 7511400.0,
+      "reward": 0.73443603515625,
+      "reward_std": 0.008873803541064262,
+      "rewards//mean": 0.73443603515625,
+      "rewards//std": 0.028051983565092087,
+      "step": 869
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.174,
+      "grad_norm": 4.21465539932251,
+      "kl": 2.301447683945298,
+      "learning_rate": 9.339615578545752e-07,
+      "loss": 0.0921,
+      "num_tokens": 7520040.0,
+      "reward": 0.7689208984375,
+      "reward_std": 0.01025544386357069,
+      "rewards//mean": 0.7689208984375,
+      "rewards//std": 0.02152203768491745,
+      "step": 870
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1742,
+      "grad_norm": 5.656636714935303,
+      "kl": 4.707288537174463,
+      "learning_rate": 9.338038518223745e-07,
+      "loss": 0.1883,
+      "num_tokens": 7528672.0,
+      "reward": 0.77557373046875,
+      "reward_std": 0.011117503046989441,
+      "rewards//mean": 0.77557373046875,
+      "rewards//std": 0.019021494314074516,
+      "step": 871
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1744,
+      "grad_norm": 6.952838897705078,
+      "kl": 3.672704292461276,
+      "learning_rate": 9.336459710540343e-07,
+      "loss": 0.1469,
+      "num_tokens": 7537480.0,
+      "reward": 0.7584228515625,
+      "reward_std": 0.01173834316432476,
+      "rewards//mean": 0.7584228515625,
+      "rewards//std": 0.027789371088147163,
+      "step": 872
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1746,
+      "grad_norm": 3.4130585193634033,
+      "kl": 0.8612559027969837,
+      "learning_rate": 9.334879156131488e-07,
+      "loss": 0.0345,
+      "num_tokens": 7546136.0,
+      "reward": 0.78509521484375,
+      "reward_std": 0.0066023776307702065,
+      "rewards//mean": 0.78509521484375,
+      "rewards//std": 0.014854062348604202,
+      "step": 873
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1748,
+      "grad_norm": 3.6809604167938232,
+      "kl": 2.227695981040597,
+      "learning_rate": 9.333296855633827e-07,
+      "loss": 0.0891,
+      "num_tokens": 7554760.0,
+      "reward": 0.77325439453125,
+      "reward_std": 0.006009181495755911,
+      "rewards//mean": 0.77325439453125,
+      "rewards//std": 0.01577049307525158,
+      "step": 874
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.175,
+      "grad_norm": 6.711003303527832,
+      "kl": 0.94081418402493,
+      "learning_rate": 9.331712809684711e-07,
+      "loss": 0.0376,
+      "num_tokens": 7563360.0,
+      "reward": 0.78240966796875,
+      "reward_std": 0.006355063058435917,
+      "rewards//mean": 0.78240966796875,
+      "rewards//std": 0.02656359225511551,
+      "step": 875
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1752,
+      "grad_norm": 4.049739360809326,
+      "kl": 2.161626299843192,
+      "learning_rate": 9.330127018922193e-07,
+      "loss": 0.0865,
+      "num_tokens": 7572032.0,
+      "reward": 0.75213623046875,
+      "reward_std": 0.008111530914902687,
+      "rewards//mean": 0.75213623046875,
+      "rewards//std": 0.024724205955863,
+      "step": 876
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1754,
+      "grad_norm": 4.6504011154174805,
+      "kl": 2.31799946911633,
+      "learning_rate": 9.32853948398503e-07,
+      "loss": 0.0927,
+      "num_tokens": 7580640.0,
+      "reward": 0.73931884765625,
+      "reward_std": 0.008125009015202522,
+      "rewards//mean": 0.73931884765625,
+      "rewards//std": 0.031251877546310425,
+      "step": 877
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1756,
+      "grad_norm": 4.644308567047119,
+      "kl": 3.264233808964491,
+      "learning_rate": 9.32695020551268e-07,
+      "loss": 0.1306,
+      "num_tokens": 7589144.0,
+      "reward": 0.72784423828125,
+      "reward_std": 0.007564897648990154,
+      "rewards//mean": 0.72784423828125,
+      "rewards//std": 0.029049299657344818,
+      "step": 878
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1758,
+      "grad_norm": 8.128149032592773,
+      "kl": 2.8776080328971148,
+      "learning_rate": 9.325359184145305e-07,
+      "loss": 0.1151,
+      "num_tokens": 7597784.0,
+      "reward": 0.74053955078125,
+      "reward_std": 0.005500072613358498,
+      "rewards//mean": 0.74053955078125,
+      "rewards//std": 0.022518403828144073,
+      "step": 879
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.176,
+      "grad_norm": 6.089498519897461,
+      "kl": 3.266717841848731,
+      "learning_rate": 9.323766420523767e-07,
+      "loss": 0.1307,
+      "num_tokens": 7606448.0,
+      "reward": 0.75592041015625,
+      "reward_std": 0.008798999711871147,
+      "rewards//mean": 0.75592041015625,
+      "rewards//std": 0.022249925881624222,
+      "step": 880
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1762,
+      "grad_norm": 7.295170307159424,
+      "kl": 4.568239964544773,
+      "learning_rate": 9.322171915289633e-07,
+      "loss": 0.1827,
+      "num_tokens": 7615096.0,
+      "reward": 0.74615478515625,
+      "reward_std": 0.010985281318426132,
+      "rewards//mean": 0.74615478515625,
+      "rewards//std": 0.023318275809288025,
+      "step": 881
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1764,
+      "grad_norm": 12.643634796142578,
+      "kl": 8.211217939853668,
+      "learning_rate": 9.320575669085169e-07,
+      "loss": 0.3284,
+      "num_tokens": 7623656.0,
+      "reward": 0.7593994140625,
+      "reward_std": 0.01920841634273529,
+      "rewards//mean": 0.7593994140625,
+      "rewards//std": 0.0324493832886219,
+      "step": 882
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1766,
+      "grad_norm": 8.465526580810547,
+      "kl": 4.972064973786473,
+      "learning_rate": 9.31897768255334e-07,
+      "loss": 0.1989,
+      "num_tokens": 7632288.0,
+      "reward": 0.74188232421875,
+      "reward_std": 0.007918823510408401,
+      "rewards//mean": 0.74188232421875,
+      "rewards//std": 0.033590611070394516,
+      "step": 883
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1768,
+      "grad_norm": 6.798526763916016,
+      "kl": 3.7168193999677896,
+      "learning_rate": 9.317377956337818e-07,
+      "loss": 0.1487,
+      "num_tokens": 7640848.0,
+      "reward": 0.72265625,
+      "reward_std": 0.009313051588833332,
+      "rewards//mean": 0.72265625,
+      "rewards//std": 0.02570466510951519,
+      "step": 884
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.177,
+      "grad_norm": 4.845640182495117,
+      "kl": 3.51280864700675,
+      "learning_rate": 9.315776491082972e-07,
+      "loss": 0.1405,
+      "num_tokens": 7649536.0,
+      "reward": 0.7335205078125,
+      "reward_std": 0.013687143102288246,
+      "rewards//mean": 0.7335205078125,
+      "rewards//std": 0.02856513299047947,
+      "step": 885
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1772,
+      "grad_norm": 6.412348747253418,
+      "kl": 3.11839210242033,
+      "learning_rate": 9.314173287433872e-07,
+      "loss": 0.1247,
+      "num_tokens": 7658176.0,
+      "reward": 0.7615966796875,
+      "reward_std": 0.011410508304834366,
+      "rewards//mean": 0.7615966796875,
+      "rewards//std": 0.022755546495318413,
+      "step": 886
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1774,
+      "grad_norm": 6.040501594543457,
+      "kl": 4.250001834705472,
+      "learning_rate": 9.312568346036287e-07,
+      "loss": 0.17,
+      "num_tokens": 7666712.0,
+      "reward": 0.78076171875,
+      "reward_std": 0.017303170636296272,
+      "rewards//mean": 0.78076171875,
+      "rewards//std": 0.030333632603287697,
+      "step": 887
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1776,
+      "grad_norm": 6.827248573303223,
+      "kl": 3.988034488633275,
+      "learning_rate": 9.310961667536688e-07,
+      "loss": 0.1595,
+      "num_tokens": 7675416.0,
+      "reward": 0.75567626953125,
+      "reward_std": 0.01062617264688015,
+      "rewards//mean": 0.75567626953125,
+      "rewards//std": 0.0382918119430542,
+      "step": 888
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1778,
+      "grad_norm": 6.079413414001465,
+      "kl": 1.765782754868269,
+      "learning_rate": 9.309353252582245e-07,
+      "loss": 0.0706,
+      "num_tokens": 7684048.0,
+      "reward": 0.7513427734375,
+      "reward_std": 0.008468233980238438,
+      "rewards//mean": 0.7513427734375,
+      "rewards//std": 0.03319104388356209,
+      "step": 889
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.178,
+      "grad_norm": 5.774899959564209,
+      "kl": 3.7291163839399815,
+      "learning_rate": 9.307743101820827e-07,
+      "loss": 0.1492,
+      "num_tokens": 7692680.0,
+      "reward": 0.7694091796875,
+      "reward_std": 0.012057274580001831,
+      "rewards//mean": 0.7694091796875,
+      "rewards//std": 0.02732132002711296,
+      "step": 890
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1782,
+      "grad_norm": 4.276165962219238,
+      "kl": 2.535430883988738,
+      "learning_rate": 9.306131215901003e-07,
+      "loss": 0.1014,
+      "num_tokens": 7701288.0,
+      "reward": 0.75970458984375,
+      "reward_std": 0.010588009841740131,
+      "rewards//mean": 0.75970458984375,
+      "rewards//std": 0.03684157878160477,
+      "step": 891
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1784,
+      "grad_norm": 8.516181945800781,
+      "kl": 3.2093895692378283,
+      "learning_rate": 9.304517595472039e-07,
+      "loss": 0.1284,
+      "num_tokens": 7709928.0,
+      "reward": 0.77935791015625,
+      "reward_std": 0.01215230394154787,
+      "rewards//mean": 0.77935791015625,
+      "rewards//std": 0.027515863999724388,
+      "step": 892
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1786,
+      "grad_norm": 4.359760284423828,
+      "kl": 3.3796353228390217,
+      "learning_rate": 9.302902241183903e-07,
+      "loss": 0.1352,
+      "num_tokens": 7718504.0,
+      "reward": 0.76055908203125,
+      "reward_std": 0.01085734460502863,
+      "rewards//mean": 0.76055908203125,
+      "rewards//std": 0.019657080993056297,
+      "step": 893
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1788,
+      "grad_norm": 3.341512680053711,
+      "kl": 2.6856060586869717,
+      "learning_rate": 9.301285153687259e-07,
+      "loss": 0.1074,
+      "num_tokens": 7727272.0,
+      "reward": 0.76300048828125,
+      "reward_std": 0.009410968981683254,
+      "rewards//mean": 0.76300048828125,
+      "rewards//std": 0.027410585433244705,
+      "step": 894
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.179,
+      "grad_norm": 7.049658298492432,
+      "kl": 4.035892087966204,
+      "learning_rate": 9.29966633363347e-07,
+      "loss": 0.1614,
+      "num_tokens": 7736008.0,
+      "reward": 0.73785400390625,
+      "reward_std": 0.007529266644269228,
+      "rewards//mean": 0.73785400390625,
+      "rewards//std": 0.036212481558322906,
+      "step": 895
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1792,
+      "grad_norm": 9.364948272705078,
+      "kl": 3.315326800569892,
+      "learning_rate": 9.298045781674595e-07,
+      "loss": 0.1326,
+      "num_tokens": 7744680.0,
+      "reward": 0.74951171875,
+      "reward_std": 0.008564095944166183,
+      "rewards//mean": 0.74951171875,
+      "rewards//std": 0.030667154118418694,
+      "step": 896
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1794,
+      "grad_norm": 4.9357123374938965,
+      "kl": 3.4647033978253603,
+      "learning_rate": 9.296423498463395e-07,
+      "loss": 0.1386,
+      "num_tokens": 7753328.0,
+      "reward": 0.78826904296875,
+      "reward_std": 0.009156836196780205,
+      "rewards//mean": 0.78826904296875,
+      "rewards//std": 0.019822722300887108,
+      "step": 897
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1796,
+      "grad_norm": 5.250432968139648,
+      "kl": 3.9305388052016497,
+      "learning_rate": 9.294799484653322e-07,
+      "loss": 0.1572,
+      "num_tokens": 7761912.0,
+      "reward": 0.77301025390625,
+      "reward_std": 0.011185125447809696,
+      "rewards//mean": 0.77301025390625,
+      "rewards//std": 0.020352492108941078,
+      "step": 898
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1798,
+      "grad_norm": 4.539963722229004,
+      "kl": 2.059523830190301,
+      "learning_rate": 9.29317374089853e-07,
+      "loss": 0.0824,
+      "num_tokens": 7770584.0,
+      "reward": 0.7725830078125,
+      "reward_std": 0.010079247877001762,
+      "rewards//mean": 0.7725830078125,
+      "rewards//std": 0.02816380187869072,
+      "step": 899
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.18,
+      "grad_norm": 5.043696880340576,
+      "kl": 4.320511316880584,
+      "learning_rate": 9.291546267853869e-07,
+      "loss": 0.1728,
+      "num_tokens": 7779176.0,
+      "reward": 0.765869140625,
+      "reward_std": 0.01119816955178976,
+      "rewards//mean": 0.765869140625,
+      "rewards//std": 0.03090611845254898,
+      "step": 900
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1802,
+      "grad_norm": 3.6633341312408447,
+      "kl": 2.414677979424596,
+      "learning_rate": 9.289917066174885e-07,
+      "loss": 0.0966,
+      "num_tokens": 7787928.0,
+      "reward": 0.75933837890625,
+      "reward_std": 0.010336682200431824,
+      "rewards//mean": 0.75933837890625,
+      "rewards//std": 0.0272560715675354,
+      "step": 901
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1804,
+      "grad_norm": 4.289097309112549,
+      "kl": 3.825998166576028,
+      "learning_rate": 9.288286136517819e-07,
+      "loss": 0.153,
+      "num_tokens": 7796560.0,
+      "reward": 0.7718505859375,
+      "reward_std": 0.010769478045403957,
+      "rewards//mean": 0.7718505859375,
+      "rewards//std": 0.03365122526884079,
+      "step": 902
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1806,
+      "grad_norm": 6.650477409362793,
+      "kl": 3.9704781882464886,
+      "learning_rate": 9.28665347953961e-07,
+      "loss": 0.1588,
+      "num_tokens": 7805264.0,
+      "reward": 0.734375,
+      "reward_std": 0.012096371501684189,
+      "rewards//mean": 0.734375,
+      "rewards//std": 0.03403255715966225,
+      "step": 903
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1808,
+      "grad_norm": 7.73297119140625,
+      "kl": 4.305436102673411,
+      "learning_rate": 9.285019095897893e-07,
+      "loss": 0.1722,
+      "num_tokens": 7813920.0,
+      "reward": 0.7784423828125,
+      "reward_std": 0.009727489203214645,
+      "rewards//mean": 0.7784423828125,
+      "rewards//std": 0.018208811059594154,
+      "step": 904
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.181,
+      "grad_norm": 5.065776824951172,
+      "kl": 4.7629342544823885,
+      "learning_rate": 9.283382986250996e-07,
+      "loss": 0.1905,
+      "num_tokens": 7822416.0,
+      "reward": 0.715576171875,
+      "reward_std": 0.012758323922753334,
+      "rewards//mean": 0.715576171875,
+      "rewards//std": 0.0380057655274868,
+      "step": 905
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1812,
+      "grad_norm": 3.914884090423584,
+      "kl": 3.7148711811751127,
+      "learning_rate": 9.281745151257945e-07,
+      "loss": 0.1486,
+      "num_tokens": 7831000.0,
+      "reward": 0.7301025390625,
+      "reward_std": 0.010050761513411999,
+      "rewards//mean": 0.7301025390625,
+      "rewards//std": 0.033412866294384,
+      "step": 906
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1814,
+      "grad_norm": 3.7434592247009277,
+      "kl": 1.0109682343900204,
+      "learning_rate": 9.280105591578458e-07,
+      "loss": 0.0404,
+      "num_tokens": 7839616.0,
+      "reward": 0.77264404296875,
+      "reward_std": 0.004304264672100544,
+      "rewards//mean": 0.77264404296875,
+      "rewards//std": 0.024021128192543983,
+      "step": 907
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1816,
+      "grad_norm": 4.644937515258789,
+      "kl": 2.340283691883087,
+      "learning_rate": 9.278464307872951e-07,
+      "loss": 0.0936,
+      "num_tokens": 7848272.0,
+      "reward": 0.78802490234375,
+      "reward_std": 0.00775968236848712,
+      "rewards//mean": 0.78802490234375,
+      "rewards//std": 0.0348847433924675,
+      "step": 908
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1818,
+      "grad_norm": 4.306415557861328,
+      "kl": 1.1228253487497568,
+      "learning_rate": 9.276821300802533e-07,
+      "loss": 0.0449,
+      "num_tokens": 7856824.0,
+      "reward": 0.77325439453125,
+      "reward_std": 0.009302186779677868,
+      "rewards//mean": 0.77325439453125,
+      "rewards//std": 0.018797343596816063,
+      "step": 909
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.182,
+      "grad_norm": 10.667795181274414,
+      "kl": 5.042736670002341,
+      "learning_rate": 9.275176571029006e-07,
+      "loss": 0.2017,
+      "num_tokens": 7865384.0,
+      "reward": 0.7467041015625,
+      "reward_std": 0.0096443435177207,
+      "rewards//mean": 0.7467041015625,
+      "rewards//std": 0.03261689096689224,
+      "step": 910
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1822,
+      "grad_norm": 3.2729358673095703,
+      "kl": 3.073277685791254,
+      "learning_rate": 9.273530119214867e-07,
+      "loss": 0.1229,
+      "num_tokens": 7874016.0,
+      "reward": 0.7843017578125,
+      "reward_std": 0.008409148082137108,
+      "rewards//mean": 0.7843017578125,
+      "rewards//std": 0.020954687148332596,
+      "step": 911
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1824,
+      "grad_norm": 4.72935676574707,
+      "kl": 3.132097691297531,
+      "learning_rate": 9.271881946023308e-07,
+      "loss": 0.1253,
+      "num_tokens": 7882688.0,
+      "reward": 0.78680419921875,
+      "reward_std": 0.01133975014090538,
+      "rewards//mean": 0.78680419921875,
+      "rewards//std": 0.025956004858016968,
+      "step": 912
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1826,
+      "grad_norm": 6.119086265563965,
+      "kl": 3.040343400090933,
+      "learning_rate": 9.270232052118212e-07,
+      "loss": 0.1216,
+      "num_tokens": 7891408.0,
+      "reward": 0.766845703125,
+      "reward_std": 0.011883992701768875,
+      "rewards//mean": 0.766845703125,
+      "rewards//std": 0.024918828159570694,
+      "step": 913
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1828,
+      "grad_norm": 9.933708190917969,
+      "kl": 2.188705800101161,
+      "learning_rate": 9.268580438164155e-07,
+      "loss": 0.0875,
+      "num_tokens": 7900024.0,
+      "reward": 0.7637939453125,
+      "reward_std": 0.007796712219715118,
+      "rewards//mean": 0.7637939453125,
+      "rewards//std": 0.029773274436593056,
+      "step": 914
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.183,
+      "grad_norm": 6.544854164123535,
+      "kl": 2.9327501095831394,
+      "learning_rate": 9.266927104826408e-07,
+      "loss": 0.1173,
+      "num_tokens": 7908696.0,
+      "reward": 0.77923583984375,
+      "reward_std": 0.012672207318246365,
+      "rewards//mean": 0.77923583984375,
+      "rewards//std": 0.030726268887519836,
+      "step": 915
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1832,
+      "grad_norm": 9.247836112976074,
+      "kl": 3.3677007276564837,
+      "learning_rate": 9.265272052770935e-07,
+      "loss": 0.1347,
+      "num_tokens": 7917392.0,
+      "reward": 0.79522705078125,
+      "reward_std": 0.011345906183123589,
+      "rewards//mean": 0.79522705078125,
+      "rewards//std": 0.019485371187329292,
+      "step": 916
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1834,
+      "grad_norm": 6.582352161407471,
+      "kl": 2.0685936380177736,
+      "learning_rate": 9.263615282664388e-07,
+      "loss": 0.0827,
+      "num_tokens": 7926048.0,
+      "reward": 0.748046875,
+      "reward_std": 0.006280519068241119,
+      "rewards//mean": 0.748046875,
+      "rewards//std": 0.04136914759874344,
+      "step": 917
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1836,
+      "grad_norm": 16.554576873779297,
+      "kl": 7.73712888173759,
+      "learning_rate": 9.261956795174115e-07,
+      "loss": 0.3095,
+      "num_tokens": 7935016.0,
+      "reward": 0.7740478515625,
+      "reward_std": 0.011473117396235466,
+      "rewards//mean": 0.7740478515625,
+      "rewards//std": 0.04325369372963905,
+      "step": 918
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1838,
+      "grad_norm": 5.420589923858643,
+      "kl": 2.9603129997849464,
+      "learning_rate": 9.260296590968156e-07,
+      "loss": 0.1184,
+      "num_tokens": 7943656.0,
+      "reward": 0.7490234375,
+      "reward_std": 0.0086138267070055,
+      "rewards//mean": 0.7490234375,
+      "rewards//std": 0.028510231524705887,
+      "step": 919
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.184,
+      "grad_norm": 3.921104669570923,
+      "kl": 1.8178562615066767,
+      "learning_rate": 9.258634670715237e-07,
+      "loss": 0.0727,
+      "num_tokens": 7952288.0,
+      "reward": 0.743408203125,
+      "reward_std": 0.01061804685741663,
+      "rewards//mean": 0.743408203125,
+      "rewards//std": 0.028526155278086662,
+      "step": 920
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1842,
+      "grad_norm": 9.220185279846191,
+      "kl": 3.6572155449539423,
+      "learning_rate": 9.256971035084784e-07,
+      "loss": 0.1463,
+      "num_tokens": 7960904.0,
+      "reward": 0.74310302734375,
+      "reward_std": 0.0070690978318452835,
+      "rewards//mean": 0.74310302734375,
+      "rewards//std": 0.037472233176231384,
+      "step": 921
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1844,
+      "grad_norm": 6.308659553527832,
+      "kl": 6.336701916530728,
+      "learning_rate": 9.255305684746907e-07,
+      "loss": 0.2535,
+      "num_tokens": 7969416.0,
+      "reward": 0.7818603515625,
+      "reward_std": 0.015000980347394943,
+      "rewards//mean": 0.7818603515625,
+      "rewards//std": 0.022186988964676857,
+      "step": 922
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1846,
+      "grad_norm": 10.97612190246582,
+      "kl": 3.909780852496624,
+      "learning_rate": 9.253638620372408e-07,
+      "loss": 0.1564,
+      "num_tokens": 7978048.0,
+      "reward": 0.77264404296875,
+      "reward_std": 0.009893986396491528,
+      "rewards//mean": 0.77264404296875,
+      "rewards//std": 0.028040649369359016,
+      "step": 923
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1848,
+      "grad_norm": 7.256546974182129,
+      "kl": 5.387146627530456,
+      "learning_rate": 9.251969842632783e-07,
+      "loss": 0.2155,
+      "num_tokens": 7986648.0,
+      "reward": 0.765380859375,
+      "reward_std": 0.010658979415893555,
+      "rewards//mean": 0.765380859375,
+      "rewards//std": 0.020266123116016388,
+      "step": 924
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.185,
+      "grad_norm": 6.968551158905029,
+      "kl": 5.536201927810907,
+      "learning_rate": 9.250299352200212e-07,
+      "loss": 0.2214,
+      "num_tokens": 7995320.0,
+      "reward": 0.7738037109375,
+      "reward_std": 0.014981362968683243,
+      "rewards//mean": 0.7738037109375,
+      "rewards//std": 0.025607606396079063,
+      "step": 925
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1852,
+      "grad_norm": 9.227314949035645,
+      "kl": 2.338663136586547,
+      "learning_rate": 9.248627149747572e-07,
+      "loss": 0.0935,
+      "num_tokens": 8003952.0,
+      "reward": 0.77825927734375,
+      "reward_std": 0.010928423143923283,
+      "rewards//mean": 0.77825927734375,
+      "rewards//std": 0.03368736431002617,
+      "step": 926
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1854,
+      "grad_norm": 10.275461196899414,
+      "kl": 5.7213707230985165,
+      "learning_rate": 9.246953235948422e-07,
+      "loss": 0.2289,
+      "num_tokens": 8012560.0,
+      "reward": 0.7593994140625,
+      "reward_std": 0.012199217453598976,
+      "rewards//mean": 0.7593994140625,
+      "rewards//std": 0.031053440645337105,
+      "step": 927
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1856,
+      "grad_norm": 5.597161293029785,
+      "kl": 4.975529843941331,
+      "learning_rate": 9.245277611477018e-07,
+      "loss": 0.199,
+      "num_tokens": 8021216.0,
+      "reward": 0.77362060546875,
+      "reward_std": 0.009361838921904564,
+      "rewards//mean": 0.77362060546875,
+      "rewards//std": 0.024144329130649567,
+      "step": 928
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1858,
+      "grad_norm": 14.230475425720215,
+      "kl": 6.894909713417292,
+      "learning_rate": 9.2436002770083e-07,
+      "loss": 0.2758,
+      "num_tokens": 8029968.0,
+      "reward": 0.7471923828125,
+      "reward_std": 0.01625603809952736,
+      "rewards//mean": 0.7471923828125,
+      "rewards//std": 0.03861016035079956,
+      "step": 929
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.186,
+      "grad_norm": 10.567397117614746,
+      "kl": 4.905621299520135,
+      "learning_rate": 9.241921233217897e-07,
+      "loss": 0.1962,
+      "num_tokens": 8038632.0,
+      "reward": 0.75860595703125,
+      "reward_std": 0.010758135467767715,
+      "rewards//mean": 0.75860595703125,
+      "rewards//std": 0.028630439192056656,
+      "step": 930
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1862,
+      "grad_norm": 12.504945755004883,
+      "kl": 6.215815335512161,
+      "learning_rate": 9.240240480782129e-07,
+      "loss": 0.2486,
+      "num_tokens": 8047248.0,
+      "reward": 0.7353515625,
+      "reward_std": 0.01204628311097622,
+      "rewards//mean": 0.7353515625,
+      "rewards//std": 0.024842167273163795,
+      "step": 931
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1864,
+      "grad_norm": 9.700063705444336,
+      "kl": 2.4597761891782284,
+      "learning_rate": 9.238558020378003e-07,
+      "loss": 0.0984,
+      "num_tokens": 8055896.0,
+      "reward": 0.73919677734375,
+      "reward_std": 0.010735702700912952,
+      "rewards//mean": 0.73919677734375,
+      "rewards//std": 0.0354938767850399,
+      "step": 932
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1866,
+      "grad_norm": 11.815288543701172,
+      "kl": 2.485420987010002,
+      "learning_rate": 9.236873852683212e-07,
+      "loss": 0.0994,
+      "num_tokens": 8064552.0,
+      "reward": 0.783447265625,
+      "reward_std": 0.009777087718248367,
+      "rewards//mean": 0.783447265625,
+      "rewards//std": 0.024704068899154663,
+      "step": 933
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1868,
+      "grad_norm": 9.662408828735352,
+      "kl": 6.564951363950968,
+      "learning_rate": 9.235187978376141e-07,
+      "loss": 0.2626,
+      "num_tokens": 8073232.0,
+      "reward": 0.75762939453125,
+      "reward_std": 0.011026167310774326,
+      "rewards//mean": 0.75762939453125,
+      "rewards//std": 0.03184446692466736,
+      "step": 934
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.187,
+      "grad_norm": 8.485936164855957,
+      "kl": 4.204109560698271,
+      "learning_rate": 9.233500398135858e-07,
+      "loss": 0.1682,
+      "num_tokens": 8081840.0,
+      "reward": 0.75640869140625,
+      "reward_std": 0.018468102440238,
+      "rewards//mean": 0.75640869140625,
+      "rewards//std": 0.031229589134454727,
+      "step": 935
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1872,
+      "grad_norm": 6.324738025665283,
+      "kl": 5.124746752902865,
+      "learning_rate": 9.23181111264212e-07,
+      "loss": 0.205,
+      "num_tokens": 8090544.0,
+      "reward": 0.750244140625,
+      "reward_std": 0.014276128262281418,
+      "rewards//mean": 0.750244140625,
+      "rewards//std": 0.034024547785520554,
+      "step": 936
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1874,
+      "grad_norm": 4.18878173828125,
+      "kl": 2.7625075317919254,
+      "learning_rate": 9.230120122575375e-07,
+      "loss": 0.1105,
+      "num_tokens": 8099208.0,
+      "reward": 0.7501220703125,
+      "reward_std": 0.009567821398377419,
+      "rewards//mean": 0.7501220703125,
+      "rewards//std": 0.024079428985714912,
+      "step": 937
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1876,
+      "grad_norm": 11.401058197021484,
+      "kl": 4.1268694922327995,
+      "learning_rate": 9.228427428616748e-07,
+      "loss": 0.1651,
+      "num_tokens": 8107856.0,
+      "reward": 0.75201416015625,
+      "reward_std": 0.012402264401316643,
+      "rewards//mean": 0.75201416015625,
+      "rewards//std": 0.026383472606539726,
+      "step": 938
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1878,
+      "grad_norm": 10.560399055480957,
+      "kl": 2.1033124066889286,
+      "learning_rate": 9.22673303144806e-07,
+      "loss": 0.0841,
+      "num_tokens": 8116456.0,
+      "reward": 0.7196044921875,
+      "reward_std": 0.010810447856783867,
+      "rewards//mean": 0.7196044921875,
+      "rewards//std": 0.04457870125770569,
+      "step": 939
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.188,
+      "grad_norm": 5.703797817230225,
+      "kl": 4.017137309536338,
+      "learning_rate": 9.22503693175181e-07,
+      "loss": 0.1607,
+      "num_tokens": 8125016.0,
+      "reward": 0.7542724609375,
+      "reward_std": 0.011905958876013756,
+      "rewards//mean": 0.7542724609375,
+      "rewards//std": 0.037134453654289246,
+      "step": 940
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1882,
+      "grad_norm": 4.964807033538818,
+      "kl": 4.473344206809998,
+      "learning_rate": 9.223339130211192e-07,
+      "loss": 0.1789,
+      "num_tokens": 8133600.0,
+      "reward": 0.76043701171875,
+      "reward_std": 0.012273428961634636,
+      "rewards//mean": 0.76043701171875,
+      "rewards//std": 0.0338330902159214,
+      "step": 941
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1884,
+      "grad_norm": 7.53067684173584,
+      "kl": 5.051701173186302,
+      "learning_rate": 9.221639627510075e-07,
+      "loss": 0.2021,
+      "num_tokens": 8142232.0,
+      "reward": 0.76171875,
+      "reward_std": 0.017527125775814056,
+      "rewards//mean": 0.76171875,
+      "rewards//std": 0.023261159658432007,
+      "step": 942
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1886,
+      "grad_norm": 10.17851448059082,
+      "kl": 3.9920184295624495,
+      "learning_rate": 9.219938424333023e-07,
+      "loss": 0.1597,
+      "num_tokens": 8150976.0,
+      "reward": 0.74755859375,
+      "reward_std": 0.016106873750686646,
+      "rewards//mean": 0.74755859375,
+      "rewards//std": 0.02955717034637928,
+      "step": 943
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1888,
+      "grad_norm": 6.212186813354492,
+      "kl": 3.6984230782836676,
+      "learning_rate": 9.218235521365276e-07,
+      "loss": 0.1479,
+      "num_tokens": 8159640.0,
+      "reward": 0.76153564453125,
+      "reward_std": 0.011410975828766823,
+      "rewards//mean": 0.76153564453125,
+      "rewards//std": 0.045327410101890564,
+      "step": 944
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.189,
+      "grad_norm": 5.5202131271362305,
+      "kl": 2.3476297073066235,
+      "learning_rate": 9.216530919292767e-07,
+      "loss": 0.0939,
+      "num_tokens": 8168288.0,
+      "reward": 0.7337646484375,
+      "reward_std": 0.006907983683049679,
+      "rewards//mean": 0.7337646484375,
+      "rewards//std": 0.026604946702718735,
+      "step": 945
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1892,
+      "grad_norm": 5.84950590133667,
+      "kl": 4.926447322592139,
+      "learning_rate": 9.214824618802107e-07,
+      "loss": 0.1971,
+      "num_tokens": 8176928.0,
+      "reward": 0.765869140625,
+      "reward_std": 0.011790703982114792,
+      "rewards//mean": 0.765869140625,
+      "rewards//std": 0.024831196293234825,
+      "step": 946
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1894,
+      "grad_norm": 6.732231616973877,
+      "kl": 4.739807171747088,
+      "learning_rate": 9.213116620580596e-07,
+      "loss": 0.1896,
+      "num_tokens": 8185472.0,
+      "reward": 0.77294921875,
+      "reward_std": 0.019628707319498062,
+      "rewards//mean": 0.77294921875,
+      "rewards//std": 0.02937633916735649,
+      "step": 947
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1896,
+      "grad_norm": 6.770096778869629,
+      "kl": 5.924624875187874,
+      "learning_rate": 9.211406925316212e-07,
+      "loss": 0.237,
+      "num_tokens": 8194096.0,
+      "reward": 0.7716064453125,
+      "reward_std": 0.016851382330060005,
+      "rewards//mean": 0.7716064453125,
+      "rewards//std": 0.03264472633600235,
+      "step": 948
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1898,
+      "grad_norm": 5.986906051635742,
+      "kl": 4.227919224649668,
+      "learning_rate": 9.209695533697623e-07,
+      "loss": 0.1691,
+      "num_tokens": 8202632.0,
+      "reward": 0.75616455078125,
+      "reward_std": 0.01510709524154663,
+      "rewards//mean": 0.75616455078125,
+      "rewards//std": 0.01998850516974926,
+      "step": 949
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.19,
+      "grad_norm": 5.989424228668213,
+      "kl": 3.7134490106254816,
+      "learning_rate": 9.207982446414177e-07,
+      "loss": 0.1485,
+      "num_tokens": 8211200.0,
+      "reward": 0.77642822265625,
+      "reward_std": 0.015160230919718742,
+      "rewards//mean": 0.77642822265625,
+      "rewards//std": 0.02674589306116104,
+      "step": 950
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1902,
+      "grad_norm": 7.805545806884766,
+      "kl": 3.576969100162387,
+      "learning_rate": 9.206267664155906e-07,
+      "loss": 0.1431,
+      "num_tokens": 8219872.0,
+      "reward": 0.77569580078125,
+      "reward_std": 0.014061015099287033,
+      "rewards//mean": 0.77569580078125,
+      "rewards//std": 0.03300506994128227,
+      "step": 951
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1904,
+      "grad_norm": 20.467113494873047,
+      "kl": 7.740051489323378,
+      "learning_rate": 9.20455118761352e-07,
+      "loss": 0.3096,
+      "num_tokens": 8228504.0,
+      "reward": 0.75762939453125,
+      "reward_std": 0.015062106773257256,
+      "rewards//mean": 0.75762939453125,
+      "rewards//std": 0.02359641157090664,
+      "step": 952
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1906,
+      "grad_norm": 7.589361190795898,
+      "kl": 4.330275880172849,
+      "learning_rate": 9.202833017478421e-07,
+      "loss": 0.1732,
+      "num_tokens": 8237192.0,
+      "reward": 0.74267578125,
+      "reward_std": 0.017783362418413162,
+      "rewards//mean": 0.74267578125,
+      "rewards//std": 0.031570009887218475,
+      "step": 953
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1908,
+      "grad_norm": 10.500250816345215,
+      "kl": 4.117554211989045,
+      "learning_rate": 9.201113154442683e-07,
+      "loss": 0.1647,
+      "num_tokens": 8245880.0,
+      "reward": 0.736328125,
+      "reward_std": 0.00937635637819767,
+      "rewards//mean": 0.736328125,
+      "rewards//std": 0.0351114422082901,
+      "step": 954
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.191,
+      "grad_norm": 8.20156478881836,
+      "kl": 3.607573974877596,
+      "learning_rate": 9.199391599199071e-07,
+      "loss": 0.1443,
+      "num_tokens": 8254472.0,
+      "reward": 0.786376953125,
+      "reward_std": 0.012617506086826324,
+      "rewards//mean": 0.786376953125,
+      "rewards//std": 0.021450528874993324,
+      "step": 955
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1912,
+      "grad_norm": 9.058380126953125,
+      "kl": 6.485325139015913,
+      "learning_rate": 9.197668352441023e-07,
+      "loss": 0.2594,
+      "num_tokens": 8263168.0,
+      "reward": 0.76019287109375,
+      "reward_std": 0.015972504392266273,
+      "rewards//mean": 0.76019287109375,
+      "rewards//std": 0.026581251993775368,
+      "step": 956
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1914,
+      "grad_norm": 13.800029754638672,
+      "kl": 5.103701438754797,
+      "learning_rate": 9.195943414862665e-07,
+      "loss": 0.2041,
+      "num_tokens": 8271784.0,
+      "reward": 0.716552734375,
+      "reward_std": 0.009655911475419998,
+      "rewards//mean": 0.716552734375,
+      "rewards//std": 0.041520360857248306,
+      "step": 957
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1916,
+      "grad_norm": 14.575108528137207,
+      "kl": 5.376349829137325,
+      "learning_rate": 9.194216787158804e-07,
+      "loss": 0.2151,
+      "num_tokens": 8280496.0,
+      "reward": 0.74310302734375,
+      "reward_std": 0.008960509672760963,
+      "rewards//mean": 0.74310302734375,
+      "rewards//std": 0.03096770867705345,
+      "step": 958
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1918,
+      "grad_norm": 13.319855690002441,
+      "kl": 2.192217580974102,
+      "learning_rate": 9.192488470024919e-07,
+      "loss": 0.0877,
+      "num_tokens": 8289160.0,
+      "reward": 0.75482177734375,
+      "reward_std": 0.005320992320775986,
+      "rewards//mean": 0.75482177734375,
+      "rewards//std": 0.024868888780474663,
+      "step": 959
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.192,
+      "grad_norm": 4.39548921585083,
+      "kl": 5.349567232653499,
+      "learning_rate": 9.190758464157182e-07,
+      "loss": 0.214,
+      "num_tokens": 8297768.0,
+      "reward": 0.732421875,
+      "reward_std": 0.01328906137496233,
+      "rewards//mean": 0.732421875,
+      "rewards//std": 0.032429084181785583,
+      "step": 960
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1922,
+      "grad_norm": 6.398663520812988,
+      "kl": 2.967155009508133,
+      "learning_rate": 9.189026770252436e-07,
+      "loss": 0.1187,
+      "num_tokens": 8306440.0,
+      "reward": 0.74798583984375,
+      "reward_std": 0.006764195393770933,
+      "rewards//mean": 0.74798583984375,
+      "rewards//std": 0.02624541148543358,
+      "step": 961
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1924,
+      "grad_norm": 11.789690017700195,
+      "kl": 2.2300401646643877,
+      "learning_rate": 9.187293389008208e-07,
+      "loss": 0.0892,
+      "num_tokens": 8315072.0,
+      "reward": 0.75286865234375,
+      "reward_std": 0.00717154610902071,
+      "rewards//mean": 0.75286865234375,
+      "rewards//std": 0.021261893212795258,
+      "step": 962
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1926,
+      "grad_norm": 7.450940132141113,
+      "kl": 1.6545574720948935,
+      "learning_rate": 9.185558321122704e-07,
+      "loss": 0.0662,
+      "num_tokens": 8323776.0,
+      "reward": 0.76458740234375,
+      "reward_std": 0.007642059121280909,
+      "rewards//mean": 0.76458740234375,
+      "rewards//std": 0.02798931673169136,
+      "step": 963
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1928,
+      "grad_norm": 3.8259990215301514,
+      "kl": 2.7195714991539717,
+      "learning_rate": 9.183821567294808e-07,
+      "loss": 0.1088,
+      "num_tokens": 8332408.0,
+      "reward": 0.754638671875,
+      "reward_std": 0.007108988240361214,
+      "rewards//mean": 0.754638671875,
+      "rewards//std": 0.021864313632249832,
+      "step": 964
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.193,
+      "grad_norm": 10.221814155578613,
+      "kl": 6.793953355401754,
+      "learning_rate": 9.182083128224086e-07,
+      "loss": 0.2718,
+      "num_tokens": 8341096.0,
+      "reward": 0.75445556640625,
+      "reward_std": 0.013266519643366337,
+      "rewards//mean": 0.75445556640625,
+      "rewards//std": 0.02583499811589718,
+      "step": 965
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1932,
+      "grad_norm": 7.6526875495910645,
+      "kl": 2.6198825035244226,
+      "learning_rate": 9.180343004610779e-07,
+      "loss": 0.1048,
+      "num_tokens": 8349760.0,
+      "reward": 0.77655029296875,
+      "reward_std": 0.015037826262414455,
+      "rewards//mean": 0.77655029296875,
+      "rewards//std": 0.026265589520335197,
+      "step": 966
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1934,
+      "grad_norm": 4.734843730926514,
+      "kl": 2.7961494512856007,
+      "learning_rate": 9.178601197155811e-07,
+      "loss": 0.1118,
+      "num_tokens": 8358440.0,
+      "reward": 0.754638671875,
+      "reward_std": 0.007869785651564598,
+      "rewards//mean": 0.754638671875,
+      "rewards//std": 0.02464517392218113,
+      "step": 967
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1936,
+      "grad_norm": 8.340118408203125,
+      "kl": 2.958549842238426,
+      "learning_rate": 9.176857706560779e-07,
+      "loss": 0.1183,
+      "num_tokens": 8367064.0,
+      "reward": 0.7425537109375,
+      "reward_std": 0.007053266745060682,
+      "rewards//mean": 0.7425537109375,
+      "rewards//std": 0.028019385412335396,
+      "step": 968
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1938,
+      "grad_norm": 11.365989685058594,
+      "kl": 2.293816465884447,
+      "learning_rate": 9.175112533527963e-07,
+      "loss": 0.0918,
+      "num_tokens": 8375712.0,
+      "reward": 0.72906494140625,
+      "reward_std": 0.013245773501694202,
+      "rewards//mean": 0.72906494140625,
+      "rewards//std": 0.029432903975248337,
+      "step": 969
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.194,
+      "grad_norm": 6.58137845993042,
+      "kl": 5.306589435786009,
+      "learning_rate": 9.173365678760317e-07,
+      "loss": 0.2123,
+      "num_tokens": 8384464.0,
+      "reward": 0.75830078125,
+      "reward_std": 0.015954727306962013,
+      "rewards//mean": 0.75830078125,
+      "rewards//std": 0.035512689501047134,
+      "step": 970
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1942,
+      "grad_norm": 7.5659074783325195,
+      "kl": 4.431315923109651,
+      "learning_rate": 9.171617142961476e-07,
+      "loss": 0.1773,
+      "num_tokens": 8393096.0,
+      "reward": 0.76904296875,
+      "reward_std": 0.015092447400093079,
+      "rewards//mean": 0.76904296875,
+      "rewards//std": 0.02833554334938526,
+      "step": 971
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1944,
+      "grad_norm": 7.9966936111450195,
+      "kl": 4.54252371750772,
+      "learning_rate": 9.169866926835747e-07,
+      "loss": 0.1817,
+      "num_tokens": 8401712.0,
+      "reward": 0.746337890625,
+      "reward_std": 0.012072852812707424,
+      "rewards//mean": 0.746337890625,
+      "rewards//std": 0.021143468096852303,
+      "step": 972
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1946,
+      "grad_norm": 6.257865905761719,
+      "kl": 3.8888294734060764,
+      "learning_rate": 9.16811503108812e-07,
+      "loss": 0.1556,
+      "num_tokens": 8410496.0,
+      "reward": 0.74932861328125,
+      "reward_std": 0.010210035368800163,
+      "rewards//mean": 0.74932861328125,
+      "rewards//std": 0.02372054196894169,
+      "step": 973
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1948,
+      "grad_norm": 12.508591651916504,
+      "kl": 6.170948276296258,
+      "learning_rate": 9.166361456424257e-07,
+      "loss": 0.2468,
+      "num_tokens": 8419064.0,
+      "reward": 0.75628662109375,
+      "reward_std": 0.012421725317835808,
+      "rewards//mean": 0.75628662109375,
+      "rewards//std": 0.03375828638672829,
+      "step": 974
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.195,
+      "grad_norm": 9.373515129089355,
+      "kl": 6.465175721794367,
+      "learning_rate": 9.164606203550497e-07,
+      "loss": 0.2586,
+      "num_tokens": 8427840.0,
+      "reward": 0.76715087890625,
+      "reward_std": 0.010892903432250023,
+      "rewards//mean": 0.76715087890625,
+      "rewards//std": 0.02707776427268982,
+      "step": 975
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1952,
+      "grad_norm": 16.16870880126953,
+      "kl": 8.718793712556362,
+      "learning_rate": 9.162849273173856e-07,
+      "loss": 0.3488,
+      "num_tokens": 8436616.0,
+      "reward": 0.772216796875,
+      "reward_std": 0.018080944195389748,
+      "rewards//mean": 0.772216796875,
+      "rewards//std": 0.03033662587404251,
+      "step": 976
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1954,
+      "grad_norm": 5.9699835777282715,
+      "kl": 4.7948022577911615,
+      "learning_rate": 9.161090666002027e-07,
+      "loss": 0.1918,
+      "num_tokens": 8445216.0,
+      "reward": 0.78094482421875,
+      "reward_std": 0.013379577547311783,
+      "rewards//mean": 0.78094482421875,
+      "rewards//std": 0.03297937288880348,
+      "step": 977
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1956,
+      "grad_norm": 5.532942295074463,
+      "kl": 5.0951950047165155,
+      "learning_rate": 9.159330382743373e-07,
+      "loss": 0.2038,
+      "num_tokens": 8453896.0,
+      "reward": 0.7803955078125,
+      "reward_std": 0.01595272868871689,
+      "rewards//mean": 0.7803955078125,
+      "rewards//std": 0.03219272568821907,
+      "step": 978
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1958,
+      "grad_norm": 9.143082618713379,
+      "kl": 5.502403872087598,
+      "learning_rate": 9.157568424106941e-07,
+      "loss": 0.2201,
+      "num_tokens": 8462552.0,
+      "reward": 0.77056884765625,
+      "reward_std": 0.013268399983644485,
+      "rewards//mean": 0.77056884765625,
+      "rewards//std": 0.02603461965918541,
+      "step": 979
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.196,
+      "grad_norm": 21.102041244506836,
+      "kl": 8.055116983130574,
+      "learning_rate": 9.155804790802443e-07,
+      "loss": 0.3222,
+      "num_tokens": 8471232.0,
+      "reward": 0.75640869140625,
+      "reward_std": 0.021582897752523422,
+      "rewards//mean": 0.75640869140625,
+      "rewards//std": 0.036334339529275894,
+      "step": 980
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1962,
+      "grad_norm": 8.947332382202148,
+      "kl": 6.066417574882507,
+      "learning_rate": 9.154039483540272e-07,
+      "loss": 0.2427,
+      "num_tokens": 8479856.0,
+      "reward": 0.76123046875,
+      "reward_std": 0.014370684511959553,
+      "rewards//mean": 0.76123046875,
+      "rewards//std": 0.026213163509964943,
+      "step": 981
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1964,
+      "grad_norm": 5.394735336303711,
+      "kl": 6.074656603857875,
+      "learning_rate": 9.152272503031495e-07,
+      "loss": 0.243,
+      "num_tokens": 8488528.0,
+      "reward": 0.74566650390625,
+      "reward_std": 0.017375901341438293,
+      "rewards//mean": 0.74566650390625,
+      "rewards//std": 0.03097064234316349,
+      "step": 982
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1966,
+      "grad_norm": 7.742849826812744,
+      "kl": 4.014471486210823,
+      "learning_rate": 9.150503849987851e-07,
+      "loss": 0.1606,
+      "num_tokens": 8497112.0,
+      "reward": 0.74835205078125,
+      "reward_std": 0.009470732882618904,
+      "rewards//mean": 0.74835205078125,
+      "rewards//std": 0.02913879044353962,
+      "step": 983
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1968,
+      "grad_norm": 5.740931034088135,
+      "kl": 2.7029154244810343,
+      "learning_rate": 9.14873352512175e-07,
+      "loss": 0.1081,
+      "num_tokens": 8505752.0,
+      "reward": 0.770263671875,
+      "reward_std": 0.010124212130904198,
+      "rewards//mean": 0.770263671875,
+      "rewards//std": 0.030890440568327904,
+      "step": 984
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.197,
+      "grad_norm": 22.27727508544922,
+      "kl": 2.468091754242778,
+      "learning_rate": 9.146961529146284e-07,
+      "loss": 0.0987,
+      "num_tokens": 8514376.0,
+      "reward": 0.7520751953125,
+      "reward_std": 0.008476316928863525,
+      "rewards//mean": 0.7520751953125,
+      "rewards//std": 0.03124200738966465,
+      "step": 985
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1972,
+      "grad_norm": 5.4475297927856445,
+      "kl": 4.27043379843235,
+      "learning_rate": 9.145187862775208e-07,
+      "loss": 0.1708,
+      "num_tokens": 8522960.0,
+      "reward": 0.730712890625,
+      "reward_std": 0.009307174943387508,
+      "rewards//mean": 0.730712890625,
+      "rewards//std": 0.02970735915005207,
+      "step": 986
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1974,
+      "grad_norm": 5.541208744049072,
+      "kl": 2.2829291969537735,
+      "learning_rate": 9.143412526722958e-07,
+      "loss": 0.0913,
+      "num_tokens": 8531496.0,
+      "reward": 0.7393798828125,
+      "reward_std": 0.010197298601269722,
+      "rewards//mean": 0.7393798828125,
+      "rewards//std": 0.03410697728395462,
+      "step": 987
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1976,
+      "grad_norm": 6.083037376403809,
+      "kl": 5.546741595491767,
+      "learning_rate": 9.141635521704636e-07,
+      "loss": 0.2219,
+      "num_tokens": 8540232.0,
+      "reward": 0.75115966796875,
+      "reward_std": 0.013699512928724289,
+      "rewards//mean": 0.75115966796875,
+      "rewards//std": 0.03273149952292442,
+      "step": 988
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1978,
+      "grad_norm": 6.53829288482666,
+      "kl": 2.6669446397572756,
+      "learning_rate": 9.139856848436023e-07,
+      "loss": 0.1067,
+      "num_tokens": 8548920.0,
+      "reward": 0.7674560546875,
+      "reward_std": 0.010561926290392876,
+      "rewards//mean": 0.7674560546875,
+      "rewards//std": 0.030694536864757538,
+      "step": 989
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.198,
+      "grad_norm": 6.94627571105957,
+      "kl": 5.959777383133769,
+      "learning_rate": 9.138076507633565e-07,
+      "loss": 0.2384,
+      "num_tokens": 8557688.0,
+      "reward": 0.77606201171875,
+      "reward_std": 0.016999781131744385,
+      "rewards//mean": 0.77606201171875,
+      "rewards//std": 0.03706483542919159,
+      "step": 990
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1982,
+      "grad_norm": 4.428110122680664,
+      "kl": 1.7214825954288244,
+      "learning_rate": 9.136294500014385e-07,
+      "loss": 0.0689,
+      "num_tokens": 8566352.0,
+      "reward": 0.76226806640625,
+      "reward_std": 0.007398197427392006,
+      "rewards//mean": 0.76226806640625,
+      "rewards//std": 0.023050576448440552,
+      "step": 991
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1984,
+      "grad_norm": 7.641226291656494,
+      "kl": 4.1124687902629375,
+      "learning_rate": 9.134510826296276e-07,
+      "loss": 0.1645,
+      "num_tokens": 8575064.0,
+      "reward": 0.7491455078125,
+      "reward_std": 0.008870312944054604,
+      "rewards//mean": 0.7491455078125,
+      "rewards//std": 0.025424884632229805,
+      "step": 992
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1986,
+      "grad_norm": 4.991622447967529,
+      "kl": 4.2028822265565395,
+      "learning_rate": 9.1327254871977e-07,
+      "loss": 0.1681,
+      "num_tokens": 8583664.0,
+      "reward": 0.74114990234375,
+      "reward_std": 0.008776310831308365,
+      "rewards//mean": 0.74114990234375,
+      "rewards//std": 0.024421734735369682,
+      "step": 993
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1988,
+      "grad_norm": 6.3080644607543945,
+      "kl": 4.167820289731026,
+      "learning_rate": 9.130938483437791e-07,
+      "loss": 0.1667,
+      "num_tokens": 8592344.0,
+      "reward": 0.75457763671875,
+      "reward_std": 0.007406790740787983,
+      "rewards//mean": 0.75457763671875,
+      "rewards//std": 0.02382243052124977,
+      "step": 994
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.199,
+      "grad_norm": 7.225706100463867,
+      "kl": 2.1177551988512278,
+      "learning_rate": 9.129149815736357e-07,
+      "loss": 0.0847,
+      "num_tokens": 8601024.0,
+      "reward": 0.73858642578125,
+      "reward_std": 0.0062733604572713375,
+      "rewards//mean": 0.73858642578125,
+      "rewards//std": 0.035408902913331985,
+      "step": 995
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1992,
+      "grad_norm": 5.533758640289307,
+      "kl": 3.727728743106127,
+      "learning_rate": 9.12735948481387e-07,
+      "loss": 0.1491,
+      "num_tokens": 8609688.0,
+      "reward": 0.76995849609375,
+      "reward_std": 0.009360658004879951,
+      "rewards//mean": 0.76995849609375,
+      "rewards//std": 0.03498094156384468,
+      "step": 996
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1994,
+      "grad_norm": 6.253836154937744,
+      "kl": 1.8540999237447977,
+      "learning_rate": 9.125567491391475e-07,
+      "loss": 0.0742,
+      "num_tokens": 8618400.0,
+      "reward": 0.77691650390625,
+      "reward_std": 0.00872066244482994,
+      "rewards//mean": 0.77691650390625,
+      "rewards//std": 0.028246654197573662,
+      "step": 997
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1996,
+      "grad_norm": 5.4483771324157715,
+      "kl": 4.82985270768404,
+      "learning_rate": 9.123773836190989e-07,
+      "loss": 0.1932,
+      "num_tokens": 8627216.0,
+      "reward": 0.7860107421875,
+      "reward_std": 0.016828035935759544,
+      "rewards//mean": 0.7860107421875,
+      "rewards//std": 0.029769206419587135,
+      "step": 998
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.1998,
+      "grad_norm": 6.1263957023620605,
+      "kl": 5.5002775229513645,
+      "learning_rate": 9.121978519934895e-07,
+      "loss": 0.22,
+      "num_tokens": 8635816.0,
+      "reward": 0.7213134765625,
+      "reward_std": 0.01754913106560707,
+      "rewards//mean": 0.7213134765625,
+      "rewards//std": 0.03597830981016159,
+      "step": 999
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2,
+      "grad_norm": 5.050541877746582,
+      "kl": 4.969757461920381,
+      "learning_rate": 9.120181543346346e-07,
+      "loss": 0.1988,
+      "num_tokens": 8644496.0,
+      "reward": 0.7615966796875,
+      "reward_std": 0.01377120241522789,
+      "rewards//mean": 0.7615966796875,
+      "rewards//std": 0.028236806392669678,
+      "step": 1000
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2002,
+      "grad_norm": 5.757530212402344,
+      "kl": 3.2096720319241285,
+      "learning_rate": 9.118382907149163e-07,
+      "loss": 0.1284,
+      "num_tokens": 8653088.0,
+      "reward": 0.77130126953125,
+      "reward_std": 0.010810548439621925,
+      "rewards//mean": 0.77130126953125,
+      "rewards//std": 0.0304406750947237,
+      "step": 1001
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2004,
+      "grad_norm": 11.434347152709961,
+      "kl": 5.517957452684641,
+      "learning_rate": 9.116582612067838e-07,
+      "loss": 0.2207,
+      "num_tokens": 8661776.0,
+      "reward": 0.751708984375,
+      "reward_std": 0.010216728784143925,
+      "rewards//mean": 0.751708984375,
+      "rewards//std": 0.0311636570841074,
+      "step": 1002
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2006,
+      "grad_norm": 5.935197830200195,
+      "kl": 1.692651392892003,
+      "learning_rate": 9.11478065882753e-07,
+      "loss": 0.0677,
+      "num_tokens": 8670552.0,
+      "reward": 0.777587890625,
+      "reward_std": 0.006503267213702202,
+      "rewards//mean": 0.777587890625,
+      "rewards//std": 0.0248604416847229,
+      "step": 1003
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2008,
+      "grad_norm": 9.93894100189209,
+      "kl": 1.6166835632175207,
+      "learning_rate": 9.112977048154064e-07,
+      "loss": 0.0647,
+      "num_tokens": 8679200.0,
+      "reward": 0.77734375,
+      "reward_std": 0.00589034054428339,
+      "rewards//mean": 0.77734375,
+      "rewards//std": 0.022162755951285362,
+      "step": 1004
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.201,
+      "grad_norm": 3.701009750366211,
+      "kl": 3.6509775668382645,
+      "learning_rate": 9.111171780773936e-07,
+      "loss": 0.146,
+      "num_tokens": 8687768.0,
+      "reward": 0.762451171875,
+      "reward_std": 0.01004645973443985,
+      "rewards//mean": 0.762451171875,
+      "rewards//std": 0.03575991094112396,
+      "step": 1005
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2012,
+      "grad_norm": 5.172314643859863,
+      "kl": 4.7731046769768,
+      "learning_rate": 9.109364857414305e-07,
+      "loss": 0.1909,
+      "num_tokens": 8696448.0,
+      "reward": 0.715576171875,
+      "reward_std": 0.011936701834201813,
+      "rewards//mean": 0.715576171875,
+      "rewards//std": 0.033341433852910995,
+      "step": 1006
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2014,
+      "grad_norm": 5.025206565856934,
+      "kl": 2.409999590367079,
+      "learning_rate": 9.107556278803002e-07,
+      "loss": 0.0964,
+      "num_tokens": 8705088.0,
+      "reward": 0.772216796875,
+      "reward_std": 0.012615786865353584,
+      "rewards//mean": 0.772216796875,
+      "rewards//std": 0.027142604812979698,
+      "step": 1007
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2016,
+      "grad_norm": 4.429189682006836,
+      "kl": 3.5807060841470957,
+      "learning_rate": 9.10574604566852e-07,
+      "loss": 0.1432,
+      "num_tokens": 8713744.0,
+      "reward": 0.7252197265625,
+      "reward_std": 0.010518459603190422,
+      "rewards//mean": 0.7252197265625,
+      "rewards//std": 0.029008951038122177,
+      "step": 1008
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2018,
+      "grad_norm": 6.667728424072266,
+      "kl": 4.246804840862751,
+      "learning_rate": 9.103934158740022e-07,
+      "loss": 0.1699,
+      "num_tokens": 8722456.0,
+      "reward": 0.7818603515625,
+      "reward_std": 0.010562303476035595,
+      "rewards//mean": 0.7818603515625,
+      "rewards//std": 0.022672908380627632,
+      "step": 1009
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.202,
+      "grad_norm": 3.641935348510742,
+      "kl": 2.2933216989040375,
+      "learning_rate": 9.102120618747336e-07,
+      "loss": 0.0917,
+      "num_tokens": 8731136.0,
+      "reward": 0.75347900390625,
+      "reward_std": 0.005826765671372414,
+      "rewards//mean": 0.75347900390625,
+      "rewards//std": 0.025220664218068123,
+      "step": 1010
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2022,
+      "grad_norm": 3.6189522743225098,
+      "kl": 2.471486421301961,
+      "learning_rate": 9.100305426420956e-07,
+      "loss": 0.0989,
+      "num_tokens": 8739872.0,
+      "reward": 0.7725830078125,
+      "reward_std": 0.00782520417124033,
+      "rewards//mean": 0.7725830078125,
+      "rewards//std": 0.01757252775132656,
+      "step": 1011
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2024,
+      "grad_norm": 5.1977128982543945,
+      "kl": 1.7518767956644297,
+      "learning_rate": 9.098488582492039e-07,
+      "loss": 0.0701,
+      "num_tokens": 8748488.0,
+      "reward": 0.7718505859375,
+      "reward_std": 0.005266233813017607,
+      "rewards//mean": 0.7718505859375,
+      "rewards//std": 0.020596198737621307,
+      "step": 1012
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2026,
+      "grad_norm": 6.584754467010498,
+      "kl": 4.939415320754051,
+      "learning_rate": 9.096670087692411e-07,
+      "loss": 0.1976,
+      "num_tokens": 8757128.0,
+      "reward": 0.761962890625,
+      "reward_std": 0.01434204913675785,
+      "rewards//mean": 0.761962890625,
+      "rewards//std": 0.027355927973985672,
+      "step": 1013
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2028,
+      "grad_norm": 4.593523979187012,
+      "kl": 3.1541009787470102,
+      "learning_rate": 9.094849942754563e-07,
+      "loss": 0.1262,
+      "num_tokens": 8765896.0,
+      "reward": 0.732666015625,
+      "reward_std": 0.009667712263762951,
+      "rewards//mean": 0.732666015625,
+      "rewards//std": 0.0408380888402462,
+      "step": 1014
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.203,
+      "grad_norm": 3.591948986053467,
+      "kl": 2.6722115837037563,
+      "learning_rate": 9.093028148411648e-07,
+      "loss": 0.1069,
+      "num_tokens": 8774560.0,
+      "reward": 0.76611328125,
+      "reward_std": 0.008937373757362366,
+      "rewards//mean": 0.76611328125,
+      "rewards//std": 0.024886002764105797,
+      "step": 1015
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2032,
+      "grad_norm": 4.636039733886719,
+      "kl": 4.22676838748157,
+      "learning_rate": 9.091204705397483e-07,
+      "loss": 0.1691,
+      "num_tokens": 8783240.0,
+      "reward": 0.76513671875,
+      "reward_std": 0.010582895949482918,
+      "rewards//mean": 0.76513671875,
+      "rewards//std": 0.02228807657957077,
+      "step": 1016
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2034,
+      "grad_norm": 5.955030918121338,
+      "kl": 2.6091385539621115,
+      "learning_rate": 9.089379614446553e-07,
+      "loss": 0.1044,
+      "num_tokens": 8791904.0,
+      "reward": 0.77581787109375,
+      "reward_std": 0.010004453361034393,
+      "rewards//mean": 0.77581787109375,
+      "rewards//std": 0.02522546611726284,
+      "step": 1017
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2036,
+      "grad_norm": 6.787014484405518,
+      "kl": 1.3281155284494162,
+      "learning_rate": 9.087552876294002e-07,
+      "loss": 0.0531,
+      "num_tokens": 8800440.0,
+      "reward": 0.76031494140625,
+      "reward_std": 0.007894234731793404,
+      "rewards//mean": 0.76031494140625,
+      "rewards//std": 0.025328470394015312,
+      "step": 1018
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2038,
+      "grad_norm": 4.883639335632324,
+      "kl": 4.105878921225667,
+      "learning_rate": 9.085724491675642e-07,
+      "loss": 0.1642,
+      "num_tokens": 8809040.0,
+      "reward": 0.75872802734375,
+      "reward_std": 0.01019999384880066,
+      "rewards//mean": 0.75872802734375,
+      "rewards//std": 0.03147197887301445,
+      "step": 1019
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.204,
+      "grad_norm": 9.381871223449707,
+      "kl": 5.848114220425487,
+      "learning_rate": 9.083894461327945e-07,
+      "loss": 0.2339,
+      "num_tokens": 8817712.0,
+      "reward": 0.7694091796875,
+      "reward_std": 0.011212250217795372,
+      "rewards//mean": 0.7694091796875,
+      "rewards//std": 0.035338133573532104,
+      "step": 1020
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2042,
+      "grad_norm": 11.179381370544434,
+      "kl": 4.110636135563254,
+      "learning_rate": 9.082062785988048e-07,
+      "loss": 0.1644,
+      "num_tokens": 8826296.0,
+      "reward": 0.76470947265625,
+      "reward_std": 0.011189059354364872,
+      "rewards//mean": 0.76470947265625,
+      "rewards//std": 0.02923784777522087,
+      "step": 1021
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2044,
+      "grad_norm": 8.341014862060547,
+      "kl": 4.192100854590535,
+      "learning_rate": 9.080229466393749e-07,
+      "loss": 0.1677,
+      "num_tokens": 8835024.0,
+      "reward": 0.76715087890625,
+      "reward_std": 0.007700664922595024,
+      "rewards//mean": 0.76715087890625,
+      "rewards//std": 0.02517741173505783,
+      "step": 1022
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2046,
+      "grad_norm": 10.33596420288086,
+      "kl": 6.088332811370492,
+      "learning_rate": 9.078394503283508e-07,
+      "loss": 0.2435,
+      "num_tokens": 8843712.0,
+      "reward": 0.7244873046875,
+      "reward_std": 0.010851889848709106,
+      "rewards//mean": 0.7244873046875,
+      "rewards//std": 0.024832110852003098,
+      "step": 1023
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2048,
+      "grad_norm": 13.496170043945312,
+      "kl": 2.899243412539363,
+      "learning_rate": 9.076557897396451e-07,
+      "loss": 0.116,
+      "num_tokens": 8852368.0,
+      "reward": 0.7635498046875,
+      "reward_std": 0.012002428993582726,
+      "rewards//mean": 0.7635498046875,
+      "rewards//std": 0.022487880662083626,
+      "step": 1024
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.205,
+      "grad_norm": 16.10167694091797,
+      "kl": 6.5625031385570765,
+      "learning_rate": 9.074719649472357e-07,
+      "loss": 0.2625,
+      "num_tokens": 8860928.0,
+      "reward": 0.7440185546875,
+      "reward_std": 0.014959938824176788,
+      "rewards//mean": 0.7440185546875,
+      "rewards//std": 0.03191693499684334,
+      "step": 1025
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2052,
+      "grad_norm": 8.42684268951416,
+      "kl": 3.2908788062632084,
+      "learning_rate": 9.072879760251679e-07,
+      "loss": 0.1316,
+      "num_tokens": 8869624.0,
+      "reward": 0.75390625,
+      "reward_std": 0.009384368546307087,
+      "rewards//mean": 0.75390625,
+      "rewards//std": 0.028237072750926018,
+      "step": 1026
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2054,
+      "grad_norm": 5.730315208435059,
+      "kl": 5.489510273560882,
+      "learning_rate": 9.071038230475519e-07,
+      "loss": 0.2196,
+      "num_tokens": 8878224.0,
+      "reward": 0.79742431640625,
+      "reward_std": 0.018564797937870026,
+      "rewards//mean": 0.79742431640625,
+      "rewards//std": 0.03276246786117554,
+      "step": 1027
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2056,
+      "grad_norm": 4.618313789367676,
+      "kl": 6.042632514610887,
+      "learning_rate": 9.069195060885646e-07,
+      "loss": 0.2417,
+      "num_tokens": 8886832.0,
+      "reward": 0.7794189453125,
+      "reward_std": 0.016224637627601624,
+      "rewards//mean": 0.7794189453125,
+      "rewards//std": 0.027339044958353043,
+      "step": 1028
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2058,
+      "grad_norm": 13.91735553741455,
+      "kl": 2.8623254615813494,
+      "learning_rate": 9.067350252224489e-07,
+      "loss": 0.1145,
+      "num_tokens": 8895536.0,
+      "reward": 0.76702880859375,
+      "reward_std": 0.011839576065540314,
+      "rewards//mean": 0.76702880859375,
+      "rewards//std": 0.022567424923181534,
+      "step": 1029
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.206,
+      "grad_norm": 14.280287742614746,
+      "kl": 9.39994865655899,
+      "learning_rate": 9.065503805235137e-07,
+      "loss": 0.376,
+      "num_tokens": 8904104.0,
+      "reward": 0.74761962890625,
+      "reward_std": 0.015248171985149384,
+      "rewards//mean": 0.74761962890625,
+      "rewards//std": 0.034334514290094376,
+      "step": 1030
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2062,
+      "grad_norm": 14.15838623046875,
+      "kl": 7.228221353143454,
+      "learning_rate": 9.06365572066134e-07,
+      "loss": 0.2891,
+      "num_tokens": 8912784.0,
+      "reward": 0.7474365234375,
+      "reward_std": 0.010489135049283504,
+      "rewards//mean": 0.7474365234375,
+      "rewards//std": 0.023737527430057526,
+      "step": 1031
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2064,
+      "grad_norm": 10.072168350219727,
+      "kl": 7.966044461354613,
+      "learning_rate": 9.061805999247503e-07,
+      "loss": 0.3186,
+      "num_tokens": 8921536.0,
+      "reward": 0.7689208984375,
+      "reward_std": 0.017396284267306328,
+      "rewards//mean": 0.7689208984375,
+      "rewards//std": 0.03338748589158058,
+      "step": 1032
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2066,
+      "grad_norm": 13.820401191711426,
+      "kl": 9.57500672340393,
+      "learning_rate": 9.059954641738697e-07,
+      "loss": 0.383,
+      "num_tokens": 8930136.0,
+      "reward": 0.72674560546875,
+      "reward_std": 0.01634276658296585,
+      "rewards//mean": 0.72674560546875,
+      "rewards//std": 0.031121307983994484,
+      "step": 1033
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2068,
+      "grad_norm": 14.01988410949707,
+      "kl": 7.840656926855445,
+      "learning_rate": 9.058101648880645e-07,
+      "loss": 0.3136,
+      "num_tokens": 8938864.0,
+      "reward": 0.7684326171875,
+      "reward_std": 0.012227555736899376,
+      "rewards//mean": 0.7684326171875,
+      "rewards//std": 0.0253867506980896,
+      "step": 1034
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.207,
+      "grad_norm": 10.134082794189453,
+      "kl": 5.084686573594809,
+      "learning_rate": 9.056247021419734e-07,
+      "loss": 0.2034,
+      "num_tokens": 8947568.0,
+      "reward": 0.75775146484375,
+      "reward_std": 0.017827101051807404,
+      "rewards//mean": 0.75775146484375,
+      "rewards//std": 0.031400710344314575,
+      "step": 1035
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2072,
+      "grad_norm": 11.32555103302002,
+      "kl": 4.593394014984369,
+      "learning_rate": 9.054390760103009e-07,
+      "loss": 0.1837,
+      "num_tokens": 8956272.0,
+      "reward": 0.7672119140625,
+      "reward_std": 0.017249010503292084,
+      "rewards//mean": 0.7672119140625,
+      "rewards//std": 0.02907150238752365,
+      "step": 1036
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2074,
+      "grad_norm": 14.796903610229492,
+      "kl": 7.180980386212468,
+      "learning_rate": 9.052532865678171e-07,
+      "loss": 0.2872,
+      "num_tokens": 8964968.0,
+      "reward": 0.757080078125,
+      "reward_std": 0.018639879301190376,
+      "rewards//mean": 0.757080078125,
+      "rewards//std": 0.036210861057043076,
+      "step": 1037
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2076,
+      "grad_norm": 13.861976623535156,
+      "kl": 7.610343635082245,
+      "learning_rate": 9.050673338893577e-07,
+      "loss": 0.3044,
+      "num_tokens": 8973680.0,
+      "reward": 0.7294921875,
+      "reward_std": 0.024282868951559067,
+      "rewards//mean": 0.7294921875,
+      "rewards//std": 0.04542652890086174,
+      "step": 1038
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2078,
+      "grad_norm": 8.83830451965332,
+      "kl": 6.84739294461906,
+      "learning_rate": 9.04881218049825e-07,
+      "loss": 0.2739,
+      "num_tokens": 8982352.0,
+      "reward": 0.7283935546875,
+      "reward_std": 0.018906250596046448,
+      "rewards//mean": 0.7283935546875,
+      "rewards//std": 0.03308689221739769,
+      "step": 1039
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.208,
+      "grad_norm": 12.931081771850586,
+      "kl": 3.712457677349448,
+      "learning_rate": 9.046949391241858e-07,
+      "loss": 0.1485,
+      "num_tokens": 8991008.0,
+      "reward": 0.72955322265625,
+      "reward_std": 0.01683615706861019,
+      "rewards//mean": 0.72955322265625,
+      "rewards//std": 0.03612961992621422,
+      "step": 1040
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2082,
+      "grad_norm": 9.187896728515625,
+      "kl": 7.100620772689581,
+      "learning_rate": 9.045084971874737e-07,
+      "loss": 0.284,
+      "num_tokens": 8999736.0,
+      "reward": 0.7667236328125,
+      "reward_std": 0.016009004786610603,
+      "rewards//mean": 0.7667236328125,
+      "rewards//std": 0.03237278759479523,
+      "step": 1041
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2084,
+      "grad_norm": 5.4558892250061035,
+      "kl": 3.412343100644648,
+      "learning_rate": 9.043218923147873e-07,
+      "loss": 0.1365,
+      "num_tokens": 9008384.0,
+      "reward": 0.73309326171875,
+      "reward_std": 0.013455754145979881,
+      "rewards//mean": 0.73309326171875,
+      "rewards//std": 0.0391681045293808,
+      "step": 1042
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2086,
+      "grad_norm": 7.426380157470703,
+      "kl": 4.572388235479593,
+      "learning_rate": 9.04135124581291e-07,
+      "loss": 0.1829,
+      "num_tokens": 9017096.0,
+      "reward": 0.74603271484375,
+      "reward_std": 0.01465071365237236,
+      "rewards//mean": 0.74603271484375,
+      "rewards//std": 0.030461056157946587,
+      "step": 1043
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2088,
+      "grad_norm": 8.59954833984375,
+      "kl": 4.7474064864218235,
+      "learning_rate": 9.039481940622146e-07,
+      "loss": 0.1899,
+      "num_tokens": 9025688.0,
+      "reward": 0.74676513671875,
+      "reward_std": 0.01007864996790886,
+      "rewards//mean": 0.74676513671875,
+      "rewards//std": 0.020723234862089157,
+      "step": 1044
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.209,
+      "grad_norm": 14.09008502960205,
+      "kl": 3.974072055891156,
+      "learning_rate": 9.037611008328543e-07,
+      "loss": 0.159,
+      "num_tokens": 9034344.0,
+      "reward": 0.75958251953125,
+      "reward_std": 0.017579447478055954,
+      "rewards//mean": 0.75958251953125,
+      "rewards//std": 0.040504928678274155,
+      "step": 1045
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2092,
+      "grad_norm": 12.411527633666992,
+      "kl": 4.509534182026982,
+      "learning_rate": 9.035738449685706e-07,
+      "loss": 0.1804,
+      "num_tokens": 9043032.0,
+      "reward": 0.745361328125,
+      "reward_std": 0.020337410271167755,
+      "rewards//mean": 0.745361328125,
+      "rewards//std": 0.043262965977191925,
+      "step": 1046
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2094,
+      "grad_norm": 13.162151336669922,
+      "kl": 3.1250076573342085,
+      "learning_rate": 9.033864265447906e-07,
+      "loss": 0.125,
+      "num_tokens": 9051672.0,
+      "reward": 0.7392578125,
+      "reward_std": 0.019312258809804916,
+      "rewards//mean": 0.7392578125,
+      "rewards//std": 0.02981417626142502,
+      "step": 1047
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2096,
+      "grad_norm": 10.181493759155273,
+      "kl": 1.9324559811502695,
+      "learning_rate": 9.031988456370061e-07,
+      "loss": 0.0773,
+      "num_tokens": 9060280.0,
+      "reward": 0.71868896484375,
+      "reward_std": 0.012040507048368454,
+      "rewards//mean": 0.71868896484375,
+      "rewards//std": 0.028852174058556557,
+      "step": 1048
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2098,
+      "grad_norm": 13.57272720336914,
+      "kl": 2.8750363830477,
+      "learning_rate": 9.030111023207749e-07,
+      "loss": 0.115,
+      "num_tokens": 9068992.0,
+      "reward": 0.732177734375,
+      "reward_std": 0.013641199097037315,
+      "rewards//mean": 0.732177734375,
+      "rewards//std": 0.029122747480869293,
+      "step": 1049
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.21,
+      "grad_norm": 11.630790710449219,
+      "kl": 2.7353729140013456,
+      "learning_rate": 9.028231966717198e-07,
+      "loss": 0.1094,
+      "num_tokens": 9077648.0,
+      "reward": 0.76727294921875,
+      "reward_std": 0.01553562842309475,
+      "rewards//mean": 0.76727294921875,
+      "rewards//std": 0.03606965392827988,
+      "step": 1050
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2102,
+      "grad_norm": 6.212285041809082,
+      "kl": 4.770133439451456,
+      "learning_rate": 9.026351287655293e-07,
+      "loss": 0.1908,
+      "num_tokens": 9086304.0,
+      "reward": 0.76593017578125,
+      "reward_std": 0.01928180642426014,
+      "rewards//mean": 0.76593017578125,
+      "rewards//std": 0.027439288794994354,
+      "step": 1051
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2104,
+      "grad_norm": 14.253013610839844,
+      "kl": 2.6883560456335545,
+      "learning_rate": 9.02446898677957e-07,
+      "loss": 0.1075,
+      "num_tokens": 9094952.0,
+      "reward": 0.77740478515625,
+      "reward_std": 0.007559204939752817,
+      "rewards//mean": 0.77740478515625,
+      "rewards//std": 0.01712612248957157,
+      "step": 1052
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2106,
+      "grad_norm": 7.245607376098633,
+      "kl": 2.7295682933181524,
+      "learning_rate": 9.02258506484822e-07,
+      "loss": 0.1092,
+      "num_tokens": 9103560.0,
+      "reward": 0.7664794921875,
+      "reward_std": 0.009524633176624775,
+      "rewards//mean": 0.7664794921875,
+      "rewards//std": 0.02959785796701908,
+      "step": 1053
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2108,
+      "grad_norm": 7.491701126098633,
+      "kl": 3.3947287648916245,
+      "learning_rate": 9.02069952262009e-07,
+      "loss": 0.1358,
+      "num_tokens": 9112192.0,
+      "reward": 0.7398681640625,
+      "reward_std": 0.012579179368913174,
+      "rewards//mean": 0.7398681640625,
+      "rewards//std": 0.0433906652033329,
+      "step": 1054
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.211,
+      "grad_norm": 11.61394214630127,
+      "kl": 7.261725638061762,
+      "learning_rate": 9.018812360854671e-07,
+      "loss": 0.2905,
+      "num_tokens": 9120840.0,
+      "reward": 0.7535400390625,
+      "reward_std": 0.021491069346666336,
+      "rewards//mean": 0.7535400390625,
+      "rewards//std": 0.03429994359612465,
+      "step": 1055
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2112,
+      "grad_norm": 7.258157730102539,
+      "kl": 5.469349566847086,
+      "learning_rate": 9.016923580312113e-07,
+      "loss": 0.2188,
+      "num_tokens": 9129488.0,
+      "reward": 0.7584228515625,
+      "reward_std": 0.011703206226229668,
+      "rewards//mean": 0.7584228515625,
+      "rewards//std": 0.03139088675379753,
+      "step": 1056
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2114,
+      "grad_norm": 7.042768955230713,
+      "kl": 6.2929673083126545,
+      "learning_rate": 9.015033181753218e-07,
+      "loss": 0.2517,
+      "num_tokens": 9138040.0,
+      "reward": 0.75927734375,
+      "reward_std": 0.017466634511947632,
+      "rewards//mean": 0.75927734375,
+      "rewards//std": 0.033592019230127335,
+      "step": 1057
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2116,
+      "grad_norm": 4.279386520385742,
+      "kl": 4.2685501947999,
+      "learning_rate": 9.013141165939438e-07,
+      "loss": 0.1707,
+      "num_tokens": 9146648.0,
+      "reward": 0.77880859375,
+      "reward_std": 0.013631336390972137,
+      "rewards//mean": 0.77880859375,
+      "rewards//std": 0.022846169769763947,
+      "step": 1058
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2118,
+      "grad_norm": 6.588010311126709,
+      "kl": 6.117425901815295,
+      "learning_rate": 9.011247533632875e-07,
+      "loss": 0.2447,
+      "num_tokens": 9155216.0,
+      "reward": 0.73260498046875,
+      "reward_std": 0.01622149348258972,
+      "rewards//mean": 0.73260498046875,
+      "rewards//std": 0.03554288670420647,
+      "step": 1059
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.212,
+      "grad_norm": 12.971135139465332,
+      "kl": 4.657304907217622,
+      "learning_rate": 9.009352285596285e-07,
+      "loss": 0.1863,
+      "num_tokens": 9163848.0,
+      "reward": 0.7452392578125,
+      "reward_std": 0.009879851713776588,
+      "rewards//mean": 0.7452392578125,
+      "rewards//std": 0.026249809190630913,
+      "step": 1060
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2122,
+      "grad_norm": 16.559602737426758,
+      "kl": 9.081853482872248,
+      "learning_rate": 9.007455422593075e-07,
+      "loss": 0.3633,
+      "num_tokens": 9172520.0,
+      "reward": 0.77069091796875,
+      "reward_std": 0.013468243181705475,
+      "rewards//mean": 0.77069091796875,
+      "rewards//std": 0.03465792536735535,
+      "step": 1061
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2124,
+      "grad_norm": 13.66545295715332,
+      "kl": 9.093273393809795,
+      "learning_rate": 9.0055569453873e-07,
+      "loss": 0.3637,
+      "num_tokens": 9181096.0,
+      "reward": 0.75872802734375,
+      "reward_std": 0.02095349133014679,
+      "rewards//mean": 0.75872802734375,
+      "rewards//std": 0.029730726033449173,
+      "step": 1062
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2126,
+      "grad_norm": 14.912736892700195,
+      "kl": 8.069859202951193,
+      "learning_rate": 9.003656854743666e-07,
+      "loss": 0.3228,
+      "num_tokens": 9189752.0,
+      "reward": 0.72222900390625,
+      "reward_std": 0.013820171356201172,
+      "rewards//mean": 0.72222900390625,
+      "rewards//std": 0.04395302012562752,
+      "step": 1063
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2128,
+      "grad_norm": 4.15281343460083,
+      "kl": 5.782654831185937,
+      "learning_rate": 9.00175515142753e-07,
+      "loss": 0.2313,
+      "num_tokens": 9198392.0,
+      "reward": 0.735595703125,
+      "reward_std": 0.01619836874306202,
+      "rewards//mean": 0.735595703125,
+      "rewards//std": 0.0386064350605011,
+      "step": 1064
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.213,
+      "grad_norm": 5.644056797027588,
+      "kl": 4.852701520547271,
+      "learning_rate": 8.9998518362049e-07,
+      "loss": 0.1941,
+      "num_tokens": 9207120.0,
+      "reward": 0.75128173828125,
+      "reward_std": 0.01600177399814129,
+      "rewards//mean": 0.75128173828125,
+      "rewards//std": 0.02943856082856655,
+      "step": 1065
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2132,
+      "grad_norm": 13.419784545898438,
+      "kl": 3.319156812503934,
+      "learning_rate": 8.997946909842424e-07,
+      "loss": 0.1328,
+      "num_tokens": 9215808.0,
+      "reward": 0.7642822265625,
+      "reward_std": 0.01321438979357481,
+      "rewards//mean": 0.7642822265625,
+      "rewards//std": 0.025038518011569977,
+      "step": 1066
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2134,
+      "grad_norm": 7.289127826690674,
+      "kl": 5.853924106806517,
+      "learning_rate": 8.996040373107414e-07,
+      "loss": 0.2342,
+      "num_tokens": 9224600.0,
+      "reward": 0.756591796875,
+      "reward_std": 0.011727886274456978,
+      "rewards//mean": 0.756591796875,
+      "rewards//std": 0.033392246812582016,
+      "step": 1067
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2136,
+      "grad_norm": 6.986499786376953,
+      "kl": 4.16066362336278,
+      "learning_rate": 8.994132226767819e-07,
+      "loss": 0.1664,
+      "num_tokens": 9233272.0,
+      "reward": 0.76837158203125,
+      "reward_std": 0.015098395757377148,
+      "rewards//mean": 0.76837158203125,
+      "rewards//std": 0.04439473897218704,
+      "step": 1068
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2138,
+      "grad_norm": 15.410289764404297,
+      "kl": 6.831299565732479,
+      "learning_rate": 8.992222471592239e-07,
+      "loss": 0.2733,
+      "num_tokens": 9241864.0,
+      "reward": 0.76251220703125,
+      "reward_std": 0.013419141992926598,
+      "rewards//mean": 0.76251220703125,
+      "rewards//std": 0.03551732376217842,
+      "step": 1069
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.214,
+      "grad_norm": 6.315592288970947,
+      "kl": 3.5881161894649267,
+      "learning_rate": 8.990311108349926e-07,
+      "loss": 0.1435,
+      "num_tokens": 9250568.0,
+      "reward": 0.764892578125,
+      "reward_std": 0.012547735124826431,
+      "rewards//mean": 0.764892578125,
+      "rewards//std": 0.03994462266564369,
+      "step": 1070
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2142,
+      "grad_norm": 5.48075008392334,
+      "kl": 6.089798431843519,
+      "learning_rate": 8.988398137810776e-07,
+      "loss": 0.2436,
+      "num_tokens": 9259280.0,
+      "reward": 0.750732421875,
+      "reward_std": 0.012852808460593224,
+      "rewards//mean": 0.750732421875,
+      "rewards//std": 0.018200911581516266,
+      "step": 1071
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2144,
+      "grad_norm": 6.696509838104248,
+      "kl": 5.232636630535126,
+      "learning_rate": 8.986483560745333e-07,
+      "loss": 0.2093,
+      "num_tokens": 9267928.0,
+      "reward": 0.73919677734375,
+      "reward_std": 0.01306705642491579,
+      "rewards//mean": 0.73919677734375,
+      "rewards//std": 0.02292281948029995,
+      "step": 1072
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2146,
+      "grad_norm": 11.351648330688477,
+      "kl": 3.9859825875610113,
+      "learning_rate": 8.984567377924789e-07,
+      "loss": 0.1594,
+      "num_tokens": 9276600.0,
+      "reward": 0.7408447265625,
+      "reward_std": 0.010947344824671745,
+      "rewards//mean": 0.7408447265625,
+      "rewards//std": 0.03646643087267876,
+      "step": 1073
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2148,
+      "grad_norm": 4.902538776397705,
+      "kl": 4.923521373420954,
+      "learning_rate": 8.982649590120981e-07,
+      "loss": 0.1969,
+      "num_tokens": 9285168.0,
+      "reward": 0.7689208984375,
+      "reward_std": 0.02040485292673111,
+      "rewards//mean": 0.7689208984375,
+      "rewards//std": 0.039611831307411194,
+      "step": 1074
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.215,
+      "grad_norm": 9.982236862182617,
+      "kl": 4.026648405939341,
+      "learning_rate": 8.980730198106394e-07,
+      "loss": 0.1611,
+      "num_tokens": 9293880.0,
+      "reward": 0.7222900390625,
+      "reward_std": 0.01029210351407528,
+      "rewards//mean": 0.7222900390625,
+      "rewards//std": 0.033063095062971115,
+      "step": 1075
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2152,
+      "grad_norm": 5.773685455322266,
+      "kl": 5.7916702423244715,
+      "learning_rate": 8.97880920265416e-07,
+      "loss": 0.2317,
+      "num_tokens": 9302696.0,
+      "reward": 0.75469970703125,
+      "reward_std": 0.014176889322698116,
+      "rewards//mean": 0.75469970703125,
+      "rewards//std": 0.032497573643922806,
+      "step": 1076
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2154,
+      "grad_norm": 7.009709358215332,
+      "kl": 2.574088655412197,
+      "learning_rate": 8.976886604538055e-07,
+      "loss": 0.103,
+      "num_tokens": 9311360.0,
+      "reward": 0.77276611328125,
+      "reward_std": 0.01645975187420845,
+      "rewards//mean": 0.77276611328125,
+      "rewards//std": 0.034481894224882126,
+      "step": 1077
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2156,
+      "grad_norm": 7.832144260406494,
+      "kl": 3.6851198878139257,
+      "learning_rate": 8.974962404532501e-07,
+      "loss": 0.1474,
+      "num_tokens": 9320040.0,
+      "reward": 0.77398681640625,
+      "reward_std": 0.01394729409366846,
+      "rewards//mean": 0.77398681640625,
+      "rewards//std": 0.02839844487607479,
+      "step": 1078
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2158,
+      "grad_norm": 5.432318687438965,
+      "kl": 4.033744588494301,
+      "learning_rate": 8.973036603412566e-07,
+      "loss": 0.1613,
+      "num_tokens": 9328624.0,
+      "reward": 0.77685546875,
+      "reward_std": 0.010864097625017166,
+      "rewards//mean": 0.77685546875,
+      "rewards//std": 0.02566222846508026,
+      "step": 1079
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.216,
+      "grad_norm": 7.712913990020752,
+      "kl": 5.05657709389925,
+      "learning_rate": 8.971109201953962e-07,
+      "loss": 0.2023,
+      "num_tokens": 9337216.0,
+      "reward": 0.7747802734375,
+      "reward_std": 0.018777944147586823,
+      "rewards//mean": 0.7747802734375,
+      "rewards//std": 0.028647683560848236,
+      "step": 1080
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2162,
+      "grad_norm": 7.410307884216309,
+      "kl": 4.115938827395439,
+      "learning_rate": 8.969180200933047e-07,
+      "loss": 0.1646,
+      "num_tokens": 9345800.0,
+      "reward": 0.76898193359375,
+      "reward_std": 0.020733792334794998,
+      "rewards//mean": 0.76898193359375,
+      "rewards//std": 0.03307882696390152,
+      "step": 1081
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2164,
+      "grad_norm": 4.52678918838501,
+      "kl": 3.3587297461926937,
+      "learning_rate": 8.967249601126821e-07,
+      "loss": 0.1343,
+      "num_tokens": 9354368.0,
+      "reward": 0.76116943359375,
+      "reward_std": 0.014119971543550491,
+      "rewards//mean": 0.76116943359375,
+      "rewards//std": 0.04334123060107231,
+      "step": 1082
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2166,
+      "grad_norm": 4.825457572937012,
+      "kl": 5.383301317691803,
+      "learning_rate": 8.96531740331293e-07,
+      "loss": 0.2153,
+      "num_tokens": 9363000.0,
+      "reward": 0.76666259765625,
+      "reward_std": 0.014202171936631203,
+      "rewards//mean": 0.76666259765625,
+      "rewards//std": 0.02980801835656166,
+      "step": 1083
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2168,
+      "grad_norm": 8.467222213745117,
+      "kl": 6.032531779259443,
+      "learning_rate": 8.963383608269663e-07,
+      "loss": 0.2413,
+      "num_tokens": 9371672.0,
+      "reward": 0.753173828125,
+      "reward_std": 0.012710340321063995,
+      "rewards//mean": 0.753173828125,
+      "rewards//std": 0.021919630467891693,
+      "step": 1084
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.217,
+      "grad_norm": 4.363509654998779,
+      "kl": 5.882176134735346,
+      "learning_rate": 8.961448216775953e-07,
+      "loss": 0.2353,
+      "num_tokens": 9380344.0,
+      "reward": 0.74407958984375,
+      "reward_std": 0.0107672568410635,
+      "rewards//mean": 0.74407958984375,
+      "rewards//std": 0.024567583575844765,
+      "step": 1085
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2172,
+      "grad_norm": 19.103225708007812,
+      "kl": 10.323764387518167,
+      "learning_rate": 8.959511229611375e-07,
+      "loss": 0.413,
+      "num_tokens": 9389040.0,
+      "reward": 0.76043701171875,
+      "reward_std": 0.019833790138363838,
+      "rewards//mean": 0.76043701171875,
+      "rewards//std": 0.027805369347333908,
+      "step": 1086
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2174,
+      "grad_norm": 11.866991996765137,
+      "kl": 6.114652732387185,
+      "learning_rate": 8.957572647556147e-07,
+      "loss": 0.2446,
+      "num_tokens": 9397592.0,
+      "reward": 0.7386474609375,
+      "reward_std": 0.013309924863278866,
+      "rewards//mean": 0.7386474609375,
+      "rewards//std": 0.029476908966898918,
+      "step": 1087
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2176,
+      "grad_norm": 5.4716410636901855,
+      "kl": 7.496361957862973,
+      "learning_rate": 8.95563247139113e-07,
+      "loss": 0.2999,
+      "num_tokens": 9406272.0,
+      "reward": 0.75762939453125,
+      "reward_std": 0.0194938275963068,
+      "rewards//mean": 0.75762939453125,
+      "rewards//std": 0.039723023772239685,
+      "step": 1088
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2178,
+      "grad_norm": 11.618474960327148,
+      "kl": 4.0657080095261335,
+      "learning_rate": 8.953690701897827e-07,
+      "loss": 0.1626,
+      "num_tokens": 9414848.0,
+      "reward": 0.7457275390625,
+      "reward_std": 0.021586135029792786,
+      "rewards//mean": 0.7457275390625,
+      "rewards//std": 0.029965855181217194,
+      "step": 1089
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.218,
+      "grad_norm": 12.74976634979248,
+      "kl": 7.561018029227853,
+      "learning_rate": 8.951747339858382e-07,
+      "loss": 0.3024,
+      "num_tokens": 9423448.0,
+      "reward": 0.728271484375,
+      "reward_std": 0.009536255151033401,
+      "rewards//mean": 0.728271484375,
+      "rewards//std": 0.02986186370253563,
+      "step": 1090
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2182,
+      "grad_norm": 4.4736480712890625,
+      "kl": 4.955842969939113,
+      "learning_rate": 8.94980238605558e-07,
+      "loss": 0.1982,
+      "num_tokens": 9432040.0,
+      "reward": 0.75628662109375,
+      "reward_std": 0.012968906201422215,
+      "rewards//mean": 0.75628662109375,
+      "rewards//std": 0.02879440411925316,
+      "step": 1091
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2184,
+      "grad_norm": 6.511483669281006,
+      "kl": 4.5022532027214766,
+      "learning_rate": 8.947855841272851e-07,
+      "loss": 0.1801,
+      "num_tokens": 9440632.0,
+      "reward": 0.76416015625,
+      "reward_std": 0.007533428259193897,
+      "rewards//mean": 0.76416015625,
+      "rewards//std": 0.023025689646601677,
+      "step": 1092
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2186,
+      "grad_norm": 15.795879364013672,
+      "kl": 9.1130366679281,
+      "learning_rate": 8.94590770629426e-07,
+      "loss": 0.3645,
+      "num_tokens": 9449312.0,
+      "reward": 0.69879150390625,
+      "reward_std": 0.015679676085710526,
+      "rewards//mean": 0.69879150390625,
+      "rewards//std": 0.0388331413269043,
+      "step": 1093
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2188,
+      "grad_norm": 4.858828067779541,
+      "kl": 3.203987028449774,
+      "learning_rate": 8.943957981904517e-07,
+      "loss": 0.1282,
+      "num_tokens": 9458032.0,
+      "reward": 0.763671875,
+      "reward_std": 0.006650151684880257,
+      "rewards//mean": 0.763671875,
+      "rewards//std": 0.026097411289811134,
+      "step": 1094
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.219,
+      "grad_norm": 5.840320587158203,
+      "kl": 3.567983377724886,
+      "learning_rate": 8.942006668888971e-07,
+      "loss": 0.1427,
+      "num_tokens": 9466552.0,
+      "reward": 0.75341796875,
+      "reward_std": 0.00962599366903305,
+      "rewards//mean": 0.75341796875,
+      "rewards//std": 0.025887753814458847,
+      "step": 1095
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2192,
+      "grad_norm": 9.678954124450684,
+      "kl": 6.329554399475455,
+      "learning_rate": 8.940053768033608e-07,
+      "loss": 0.2532,
+      "num_tokens": 9475248.0,
+      "reward": 0.744384765625,
+      "reward_std": 0.01467149518430233,
+      "rewards//mean": 0.744384765625,
+      "rewards//std": 0.036906491965055466,
+      "step": 1096
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2194,
+      "grad_norm": 5.616601943969727,
+      "kl": 6.981848675757647,
+      "learning_rate": 8.938099280125062e-07,
+      "loss": 0.2793,
+      "num_tokens": 9483984.0,
+      "reward": 0.77020263671875,
+      "reward_std": 0.023348022252321243,
+      "rewards//mean": 0.77020263671875,
+      "rewards//std": 0.03965055197477341,
+      "step": 1097
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2196,
+      "grad_norm": 8.251206398010254,
+      "kl": 5.501217184588313,
+      "learning_rate": 8.936143205950595e-07,
+      "loss": 0.22,
+      "num_tokens": 9492560.0,
+      "reward": 0.7432861328125,
+      "reward_std": 0.014951873570680618,
+      "rewards//mean": 0.7432861328125,
+      "rewards//std": 0.03821925073862076,
+      "step": 1098
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2198,
+      "grad_norm": 5.047215938568115,
+      "kl": 4.080555723980069,
+      "learning_rate": 8.934185546298115e-07,
+      "loss": 0.1632,
+      "num_tokens": 9501136.0,
+      "reward": 0.70928955078125,
+      "reward_std": 0.01419641263782978,
+      "rewards//mean": 0.70928955078125,
+      "rewards//std": 0.04334646835923195,
+      "step": 1099
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.22,
+      "grad_norm": 8.520955085754395,
+      "kl": 4.37362390384078,
+      "learning_rate": 8.932226301956169e-07,
+      "loss": 0.1749,
+      "num_tokens": 9509816.0,
+      "reward": 0.74566650390625,
+      "reward_std": 0.015053149312734604,
+      "rewards//mean": 0.74566650390625,
+      "rewards//std": 0.02922438271343708,
+      "step": 1100
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2202,
+      "grad_norm": 15.754254341125488,
+      "kl": 6.733986398205161,
+      "learning_rate": 8.930265473713937e-07,
+      "loss": 0.2694,
+      "num_tokens": 9518472.0,
+      "reward": 0.75933837890625,
+      "reward_std": 0.017398901283740997,
+      "rewards//mean": 0.75933837890625,
+      "rewards//std": 0.04254456236958504,
+      "step": 1101
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2204,
+      "grad_norm": 6.832894802093506,
+      "kl": 3.5568563863635063,
+      "learning_rate": 8.928303062361243e-07,
+      "loss": 0.1423,
+      "num_tokens": 9527136.0,
+      "reward": 0.7445068359375,
+      "reward_std": 0.009682252071797848,
+      "rewards//mean": 0.7445068359375,
+      "rewards//std": 0.03515086695551872,
+      "step": 1102
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2206,
+      "grad_norm": 6.307701110839844,
+      "kl": 3.7589014023542404,
+      "learning_rate": 8.926339068688545e-07,
+      "loss": 0.1504,
+      "num_tokens": 9535776.0,
+      "reward": 0.75201416015625,
+      "reward_std": 0.01643678918480873,
+      "rewards//mean": 0.75201416015625,
+      "rewards//std": 0.039432723075151443,
+      "step": 1103
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2208,
+      "grad_norm": 9.606477737426758,
+      "kl": 3.097434274852276,
+      "learning_rate": 8.924373493486941e-07,
+      "loss": 0.1239,
+      "num_tokens": 9544424.0,
+      "reward": 0.7449951171875,
+      "reward_std": 0.008526843972504139,
+      "rewards//mean": 0.7449951171875,
+      "rewards//std": 0.03196622058749199,
+      "step": 1104
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.221,
+      "grad_norm": 9.570741653442383,
+      "kl": 4.895392829552293,
+      "learning_rate": 8.922406337548161e-07,
+      "loss": 0.1958,
+      "num_tokens": 9553032.0,
+      "reward": 0.71478271484375,
+      "reward_std": 0.011852547526359558,
+      "rewards//mean": 0.71478271484375,
+      "rewards//std": 0.024934541434049606,
+      "step": 1105
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2212,
+      "grad_norm": 13.413549423217773,
+      "kl": 7.705706216394901,
+      "learning_rate": 8.920437601664579e-07,
+      "loss": 0.3082,
+      "num_tokens": 9561712.0,
+      "reward": 0.7745361328125,
+      "reward_std": 0.011381777003407478,
+      "rewards//mean": 0.7745361328125,
+      "rewards//std": 0.03425578027963638,
+      "step": 1106
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2214,
+      "grad_norm": 6.399602890014648,
+      "kl": 2.971532279625535,
+      "learning_rate": 8.918467286629198e-07,
+      "loss": 0.1189,
+      "num_tokens": 9570320.0,
+      "reward": 0.732421875,
+      "reward_std": 0.008850199170410633,
+      "rewards//mean": 0.732421875,
+      "rewards//std": 0.03154314309358597,
+      "step": 1107
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2216,
+      "grad_norm": 10.602251052856445,
+      "kl": 3.5051215067505836,
+      "learning_rate": 8.916495393235665e-07,
+      "loss": 0.1402,
+      "num_tokens": 9578848.0,
+      "reward": 0.748779296875,
+      "reward_std": 0.014799104072153568,
+      "rewards//mean": 0.748779296875,
+      "rewards//std": 0.037011343985795975,
+      "step": 1108
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2218,
+      "grad_norm": 4.765389442443848,
+      "kl": 5.119537103921175,
+      "learning_rate": 8.914521922278255e-07,
+      "loss": 0.2048,
+      "num_tokens": 9587480.0,
+      "reward": 0.73248291015625,
+      "reward_std": 0.010286146774888039,
+      "rewards//mean": 0.73248291015625,
+      "rewards//std": 0.030241606757044792,
+      "step": 1109
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.222,
+      "grad_norm": 11.750473022460938,
+      "kl": 3.9613872449845076,
+      "learning_rate": 8.912546874551882e-07,
+      "loss": 0.1585,
+      "num_tokens": 9596152.0,
+      "reward": 0.74652099609375,
+      "reward_std": 0.006439933553338051,
+      "rewards//mean": 0.74652099609375,
+      "rewards//std": 0.03288974612951279,
+      "step": 1110
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2222,
+      "grad_norm": 14.945819854736328,
+      "kl": 7.06269839219749,
+      "learning_rate": 8.910570250852096e-07,
+      "loss": 0.2825,
+      "num_tokens": 9604800.0,
+      "reward": 0.73834228515625,
+      "reward_std": 0.01386193186044693,
+      "rewards//mean": 0.73834228515625,
+      "rewards//std": 0.043678704649209976,
+      "step": 1111
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2224,
+      "grad_norm": 5.244224548339844,
+      "kl": 4.3932148311287165,
+      "learning_rate": 8.908592051975081e-07,
+      "loss": 0.1757,
+      "num_tokens": 9613480.0,
+      "reward": 0.7559814453125,
+      "reward_std": 0.013672607019543648,
+      "rewards//mean": 0.7559814453125,
+      "rewards//std": 0.02797829546034336,
+      "step": 1112
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2226,
+      "grad_norm": 8.444457054138184,
+      "kl": 3.298034707084298,
+      "learning_rate": 8.906612278717655e-07,
+      "loss": 0.1319,
+      "num_tokens": 9622088.0,
+      "reward": 0.73394775390625,
+      "reward_std": 0.014404860325157642,
+      "rewards//mean": 0.73394775390625,
+      "rewards//std": 0.038185324519872665,
+      "step": 1113
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2228,
+      "grad_norm": 12.326435089111328,
+      "kl": 5.131952002644539,
+      "learning_rate": 8.90463093187727e-07,
+      "loss": 0.2053,
+      "num_tokens": 9630704.0,
+      "reward": 0.762451171875,
+      "reward_std": 0.01745082065463066,
+      "rewards//mean": 0.762451171875,
+      "rewards//std": 0.025881905108690262,
+      "step": 1114
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.223,
+      "grad_norm": 7.397994041442871,
+      "kl": 2.4171334765851498,
+      "learning_rate": 8.902648012252012e-07,
+      "loss": 0.0967,
+      "num_tokens": 9639360.0,
+      "reward": 0.7742919921875,
+      "reward_std": 0.00961617287248373,
+      "rewards//mean": 0.7742919921875,
+      "rewards//std": 0.024601832032203674,
+      "step": 1115
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2232,
+      "grad_norm": 6.574716091156006,
+      "kl": 4.173199244774878,
+      "learning_rate": 8.900663520640603e-07,
+      "loss": 0.1669,
+      "num_tokens": 9647976.0,
+      "reward": 0.73150634765625,
+      "reward_std": 0.014405167661607265,
+      "rewards//mean": 0.73150634765625,
+      "rewards//std": 0.03448672220110893,
+      "step": 1116
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2234,
+      "grad_norm": 7.505921840667725,
+      "kl": 2.7678875029087067,
+      "learning_rate": 8.898677457842394e-07,
+      "loss": 0.1107,
+      "num_tokens": 9656608.0,
+      "reward": 0.774169921875,
+      "reward_std": 0.015123208984732628,
+      "rewards//mean": 0.774169921875,
+      "rewards//std": 0.029796907678246498,
+      "step": 1117
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2236,
+      "grad_norm": 6.814366817474365,
+      "kl": 5.7939563523977995,
+      "learning_rate": 8.896689824657371e-07,
+      "loss": 0.2318,
+      "num_tokens": 9665184.0,
+      "reward": 0.75482177734375,
+      "reward_std": 0.01692846789956093,
+      "rewards//mean": 0.75482177734375,
+      "rewards//std": 0.03000643104314804,
+      "step": 1118
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2238,
+      "grad_norm": 9.690752029418945,
+      "kl": 6.7204610873013735,
+      "learning_rate": 8.894700621886152e-07,
+      "loss": 0.2688,
+      "num_tokens": 9673856.0,
+      "reward": 0.7537841796875,
+      "reward_std": 0.01766577549278736,
+      "rewards//mean": 0.7537841796875,
+      "rewards//std": 0.039173755794763565,
+      "step": 1119
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.224,
+      "grad_norm": 16.34100914001465,
+      "kl": 3.3960287645459175,
+      "learning_rate": 8.892709850329989e-07,
+      "loss": 0.1358,
+      "num_tokens": 9682448.0,
+      "reward": 0.7314453125,
+      "reward_std": 0.011177017353475094,
+      "rewards//mean": 0.7314453125,
+      "rewards//std": 0.03624512627720833,
+      "step": 1120
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2242,
+      "grad_norm": 8.458902359008789,
+      "kl": 4.556045098230243,
+      "learning_rate": 8.890717510790762e-07,
+      "loss": 0.1822,
+      "num_tokens": 9691064.0,
+      "reward": 0.74200439453125,
+      "reward_std": 0.00906654354184866,
+      "rewards//mean": 0.74200439453125,
+      "rewards//std": 0.026697741821408272,
+      "step": 1121
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2244,
+      "grad_norm": 12.766468048095703,
+      "kl": 5.043446036055684,
+      "learning_rate": 8.888723604070989e-07,
+      "loss": 0.2017,
+      "num_tokens": 9699704.0,
+      "reward": 0.76190185546875,
+      "reward_std": 0.009677472524344921,
+      "rewards//mean": 0.76190185546875,
+      "rewards//std": 0.03258131071925163,
+      "step": 1122
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2246,
+      "grad_norm": 8.831002235412598,
+      "kl": 3.949399497359991,
+      "learning_rate": 8.886728130973813e-07,
+      "loss": 0.158,
+      "num_tokens": 9708296.0,
+      "reward": 0.77288818359375,
+      "reward_std": 0.012819640338420868,
+      "rewards//mean": 0.77288818359375,
+      "rewards//std": 0.028813323006033897,
+      "step": 1123
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2248,
+      "grad_norm": 4.6322197914123535,
+      "kl": 4.173149090260267,
+      "learning_rate": 8.884731092303011e-07,
+      "loss": 0.1669,
+      "num_tokens": 9717040.0,
+      "reward": 0.760498046875,
+      "reward_std": 0.0113409124314785,
+      "rewards//mean": 0.760498046875,
+      "rewards//std": 0.028964301571249962,
+      "step": 1124
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.225,
+      "grad_norm": 10.083248138427734,
+      "kl": 2.7000562883913517,
+      "learning_rate": 8.882732488862987e-07,
+      "loss": 0.108,
+      "num_tokens": 9725680.0,
+      "reward": 0.73944091796875,
+      "reward_std": 0.008040301501750946,
+      "rewards//mean": 0.73944091796875,
+      "rewards//std": 0.03297478333115578,
+      "step": 1125
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2252,
+      "grad_norm": 8.444313049316406,
+      "kl": 5.095565229654312,
+      "learning_rate": 8.880732321458784e-07,
+      "loss": 0.2038,
+      "num_tokens": 9734240.0,
+      "reward": 0.77410888671875,
+      "reward_std": 0.012255186215043068,
+      "rewards//mean": 0.77410888671875,
+      "rewards//std": 0.01863071136176586,
+      "step": 1126
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2254,
+      "grad_norm": 10.66525650024414,
+      "kl": 2.797660192474723,
+      "learning_rate": 8.878730590896065e-07,
+      "loss": 0.1119,
+      "num_tokens": 9742928.0,
+      "reward": 0.712890625,
+      "reward_std": 0.007319233845919371,
+      "rewards//mean": 0.712890625,
+      "rewards//std": 0.04367043077945709,
+      "step": 1127
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2256,
+      "grad_norm": 5.685501575469971,
+      "kl": 2.1741212755441666,
+      "learning_rate": 8.876727297981127e-07,
+      "loss": 0.087,
+      "num_tokens": 9751496.0,
+      "reward": 0.78173828125,
+      "reward_std": 0.009359104558825493,
+      "rewards//mean": 0.78173828125,
+      "rewards//std": 0.02638813480734825,
+      "step": 1128
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2258,
+      "grad_norm": 4.537426948547363,
+      "kl": 1.442899452522397,
+      "learning_rate": 8.874722443520898e-07,
+      "loss": 0.0577,
+      "num_tokens": 9760136.0,
+      "reward": 0.77166748046875,
+      "reward_std": 0.011438120156526566,
+      "rewards//mean": 0.77166748046875,
+      "rewards//std": 0.028797032311558723,
+      "step": 1129
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.226,
+      "grad_norm": 6.674938678741455,
+      "kl": 2.905785081908107,
+      "learning_rate": 8.872716028322931e-07,
+      "loss": 0.1162,
+      "num_tokens": 9768864.0,
+      "reward": 0.7750244140625,
+      "reward_std": 0.014743040315806866,
+      "rewards//mean": 0.7750244140625,
+      "rewards//std": 0.03611605241894722,
+      "step": 1130
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2262,
+      "grad_norm": 4.763418674468994,
+      "kl": 2.849360417574644,
+      "learning_rate": 8.870708053195413e-07,
+      "loss": 0.114,
+      "num_tokens": 9777504.0,
+      "reward": 0.73541259765625,
+      "reward_std": 0.012042814865708351,
+      "rewards//mean": 0.73541259765625,
+      "rewards//std": 0.041405677795410156,
+      "step": 1131
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2264,
+      "grad_norm": 5.136250019073486,
+      "kl": 3.0846480950713158,
+      "learning_rate": 8.868698518947151e-07,
+      "loss": 0.1234,
+      "num_tokens": 9786088.0,
+      "reward": 0.73541259765625,
+      "reward_std": 0.009864496067166328,
+      "rewards//mean": 0.73541259765625,
+      "rewards//std": 0.024873757734894753,
+      "step": 1132
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2266,
+      "grad_norm": 8.897073745727539,
+      "kl": 0.7811648137867451,
+      "learning_rate": 8.866687426387591e-07,
+      "loss": 0.0312,
+      "num_tokens": 9794704.0,
+      "reward": 0.76324462890625,
+      "reward_std": 0.00455887708812952,
+      "rewards//mean": 0.76324462890625,
+      "rewards//std": 0.030118217691779137,
+      "step": 1133
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2268,
+      "grad_norm": 11.335514068603516,
+      "kl": 4.045972425490618,
+      "learning_rate": 8.864674776326797e-07,
+      "loss": 0.1618,
+      "num_tokens": 9803360.0,
+      "reward": 0.800537109375,
+      "reward_std": 0.006927927955985069,
+      "rewards//mean": 0.800537109375,
+      "rewards//std": 0.02517027035355568,
+      "step": 1134
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.227,
+      "grad_norm": 4.795708656311035,
+      "kl": 0.9940475765615702,
+      "learning_rate": 8.862660569575464e-07,
+      "loss": 0.0398,
+      "num_tokens": 9812040.0,
+      "reward": 0.7625732421875,
+      "reward_std": 0.0066719925962388515,
+      "rewards//mean": 0.7625732421875,
+      "rewards//std": 0.02362758480012417,
+      "step": 1135
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2272,
+      "grad_norm": 4.9466633796691895,
+      "kl": 4.480608073994517,
+      "learning_rate": 8.860644806944917e-07,
+      "loss": 0.1792,
+      "num_tokens": 9820664.0,
+      "reward": 0.72784423828125,
+      "reward_std": 0.009332980960607529,
+      "rewards//mean": 0.72784423828125,
+      "rewards//std": 0.032309774309396744,
+      "step": 1136
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2274,
+      "grad_norm": 4.079766273498535,
+      "kl": 1.916475109755993,
+      "learning_rate": 8.858627489247104e-07,
+      "loss": 0.0767,
+      "num_tokens": 9829264.0,
+      "reward": 0.76568603515625,
+      "reward_std": 0.007418432272970676,
+      "rewards//mean": 0.76568603515625,
+      "rewards//std": 0.024294957518577576,
+      "step": 1137
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2276,
+      "grad_norm": 3.6407711505889893,
+      "kl": 2.09932229667902,
+      "learning_rate": 8.856608617294599e-07,
+      "loss": 0.084,
+      "num_tokens": 9837976.0,
+      "reward": 0.75091552734375,
+      "reward_std": 0.0055526187643408775,
+      "rewards//mean": 0.75091552734375,
+      "rewards//std": 0.030223581939935684,
+      "step": 1138
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2278,
+      "grad_norm": 3.612649917602539,
+      "kl": 2.2236559856683016,
+      "learning_rate": 8.854588191900604e-07,
+      "loss": 0.0889,
+      "num_tokens": 9846624.0,
+      "reward": 0.75433349609375,
+      "reward_std": 0.007318149320781231,
+      "rewards//mean": 0.75433349609375,
+      "rewards//std": 0.025412004441022873,
+      "step": 1139
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.228,
+      "grad_norm": 5.206478595733643,
+      "kl": 4.678205693140626,
+      "learning_rate": 8.852566213878946e-07,
+      "loss": 0.1871,
+      "num_tokens": 9855208.0,
+      "reward": 0.75811767578125,
+      "reward_std": 0.012443074025213718,
+      "rewards//mean": 0.75811767578125,
+      "rewards//std": 0.03624047711491585,
+      "step": 1140
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2282,
+      "grad_norm": 4.288930892944336,
+      "kl": 2.0811544209718704,
+      "learning_rate": 8.850542684044078e-07,
+      "loss": 0.0832,
+      "num_tokens": 9863816.0,
+      "reward": 0.8023681640625,
+      "reward_std": 0.006106534507125616,
+      "rewards//mean": 0.8023681640625,
+      "rewards//std": 0.0222033578902483,
+      "step": 1141
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2284,
+      "grad_norm": 4.588595867156982,
+      "kl": 3.811111245304346,
+      "learning_rate": 8.848517603211078e-07,
+      "loss": 0.1524,
+      "num_tokens": 9872392.0,
+      "reward": 0.75927734375,
+      "reward_std": 0.010999436490237713,
+      "rewards//mean": 0.75927734375,
+      "rewards//std": 0.02885223925113678,
+      "step": 1142
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2286,
+      "grad_norm": 12.965023040771484,
+      "kl": 5.332149356603622,
+      "learning_rate": 8.846490972195646e-07,
+      "loss": 0.2133,
+      "num_tokens": 9881024.0,
+      "reward": 0.73846435546875,
+      "reward_std": 0.008660933934152126,
+      "rewards//mean": 0.73846435546875,
+      "rewards//std": 0.030555827543139458,
+      "step": 1143
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2288,
+      "grad_norm": 9.358354568481445,
+      "kl": 4.364532317966223,
+      "learning_rate": 8.844462791814112e-07,
+      "loss": 0.1746,
+      "num_tokens": 9889648.0,
+      "reward": 0.74139404296875,
+      "reward_std": 0.00542416563257575,
+      "rewards//mean": 0.74139404296875,
+      "rewards//std": 0.02432173490524292,
+      "step": 1144
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.229,
+      "grad_norm": 5.2452521324157715,
+      "kl": 3.6066288389265537,
+      "learning_rate": 8.842433062883425e-07,
+      "loss": 0.1443,
+      "num_tokens": 9898224.0,
+      "reward": 0.77191162109375,
+      "reward_std": 0.015570230782032013,
+      "rewards//mean": 0.77191162109375,
+      "rewards//std": 0.02311091497540474,
+      "step": 1145
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2292,
+      "grad_norm": 10.84929370880127,
+      "kl": 4.584597546607256,
+      "learning_rate": 8.840401786221159e-07,
+      "loss": 0.1834,
+      "num_tokens": 9906824.0,
+      "reward": 0.73260498046875,
+      "reward_std": 0.007782891392707825,
+      "rewards//mean": 0.73260498046875,
+      "rewards//std": 0.027995266020298004,
+      "step": 1146
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2294,
+      "grad_norm": 7.70128870010376,
+      "kl": 2.4459972754120827,
+      "learning_rate": 8.838368962645513e-07,
+      "loss": 0.0978,
+      "num_tokens": 9915560.0,
+      "reward": 0.74884033203125,
+      "reward_std": 0.008964098989963531,
+      "rewards//mean": 0.74884033203125,
+      "rewards//std": 0.025767529383301735,
+      "step": 1147
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2296,
+      "grad_norm": 3.8446500301361084,
+      "kl": 4.139189610257745,
+      "learning_rate": 8.836334592975308e-07,
+      "loss": 0.1656,
+      "num_tokens": 9924232.0,
+      "reward": 0.71258544921875,
+      "reward_std": 0.010579858906567097,
+      "rewards//mean": 0.71258544921875,
+      "rewards//std": 0.03684691712260246,
+      "step": 1148
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2298,
+      "grad_norm": 4.000614643096924,
+      "kl": 4.734907872974873,
+      "learning_rate": 8.834298678029988e-07,
+      "loss": 0.1894,
+      "num_tokens": 9932832.0,
+      "reward": 0.7659912109375,
+      "reward_std": 0.01792335882782936,
+      "rewards//mean": 0.7659912109375,
+      "rewards//std": 0.037889882922172546,
+      "step": 1149
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.23,
+      "grad_norm": 5.773425579071045,
+      "kl": 3.126021906733513,
+      "learning_rate": 8.83226121862962e-07,
+      "loss": 0.125,
+      "num_tokens": 9941384.0,
+      "reward": 0.76690673828125,
+      "reward_std": 0.01594298519194126,
+      "rewards//mean": 0.76690673828125,
+      "rewards//std": 0.030860472470521927,
+      "step": 1150
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2302,
+      "grad_norm": 6.255007743835449,
+      "kl": 3.4385843258351088,
+      "learning_rate": 8.83022221559489e-07,
+      "loss": 0.1375,
+      "num_tokens": 9949968.0,
+      "reward": 0.751708984375,
+      "reward_std": 0.011574260890483856,
+      "rewards//mean": 0.751708984375,
+      "rewards//std": 0.024947969242930412,
+      "step": 1151
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2304,
+      "grad_norm": 6.391628265380859,
+      "kl": 3.548680679872632,
+      "learning_rate": 8.82818166974711e-07,
+      "loss": 0.1419,
+      "num_tokens": 9958640.0,
+      "reward": 0.77655029296875,
+      "reward_std": 0.012465916574001312,
+      "rewards//mean": 0.77655029296875,
+      "rewards//std": 0.026126904413104057,
+      "step": 1152
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2306,
+      "grad_norm": 5.595583438873291,
+      "kl": 7.071510713547468,
+      "learning_rate": 8.826139581908211e-07,
+      "loss": 0.2829,
+      "num_tokens": 9967248.0,
+      "reward": 0.77203369140625,
+      "reward_std": 0.0170577522367239,
+      "rewards//mean": 0.77203369140625,
+      "rewards//std": 0.03440322354435921,
+      "step": 1153
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2308,
+      "grad_norm": 3.3383522033691406,
+      "kl": 2.09860529191792,
+      "learning_rate": 8.824095952900746e-07,
+      "loss": 0.0839,
+      "num_tokens": 9975864.0,
+      "reward": 0.7640380859375,
+      "reward_std": 0.007309587672352791,
+      "rewards//mean": 0.7640380859375,
+      "rewards//std": 0.026944171637296677,
+      "step": 1154
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.231,
+      "grad_norm": 6.839156627655029,
+      "kl": 3.384455820545554,
+      "learning_rate": 8.822050783547889e-07,
+      "loss": 0.1354,
+      "num_tokens": 9984496.0,
+      "reward": 0.78863525390625,
+      "reward_std": 0.016305115073919296,
+      "rewards//mean": 0.78863525390625,
+      "rewards//std": 0.030065901577472687,
+      "step": 1155
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2312,
+      "grad_norm": 4.410799503326416,
+      "kl": 1.7257406003773212,
+      "learning_rate": 8.820004074673433e-07,
+      "loss": 0.069,
+      "num_tokens": 9993096.0,
+      "reward": 0.78155517578125,
+      "reward_std": 0.004620138555765152,
+      "rewards//mean": 0.78155517578125,
+      "rewards//std": 0.020591329783201218,
+      "step": 1156
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2314,
+      "grad_norm": 13.29317569732666,
+      "kl": 6.0627273842692375,
+      "learning_rate": 8.817955827101792e-07,
+      "loss": 0.2425,
+      "num_tokens": 10001696.0,
+      "reward": 0.7540283203125,
+      "reward_std": 0.008711063303053379,
+      "rewards//mean": 0.7540283203125,
+      "rewards//std": 0.0346529558300972,
+      "step": 1157
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2316,
+      "grad_norm": 7.899120330810547,
+      "kl": 2.4596549551934004,
+      "learning_rate": 8.815906041658001e-07,
+      "loss": 0.0984,
+      "num_tokens": 10010312.0,
+      "reward": 0.73455810546875,
+      "reward_std": 0.005697751417756081,
+      "rewards//mean": 0.73455810546875,
+      "rewards//std": 0.023973187431693077,
+      "step": 1158
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2318,
+      "grad_norm": 7.7236785888671875,
+      "kl": 3.558136461302638,
+      "learning_rate": 8.813854719167712e-07,
+      "loss": 0.1423,
+      "num_tokens": 10018944.0,
+      "reward": 0.7491455078125,
+      "reward_std": 0.005457497201859951,
+      "rewards//mean": 0.7491455078125,
+      "rewards//std": 0.02287762239575386,
+      "step": 1159
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.232,
+      "grad_norm": 4.86129903793335,
+      "kl": 3.5358568504452705,
+      "learning_rate": 8.8118018604572e-07,
+      "loss": 0.1414,
+      "num_tokens": 10027520.0,
+      "reward": 0.7706298828125,
+      "reward_std": 0.013745088130235672,
+      "rewards//mean": 0.7706298828125,
+      "rewards//std": 0.032327864319086075,
+      "step": 1160
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2322,
+      "grad_norm": 4.532315254211426,
+      "kl": 2.457659238949418,
+      "learning_rate": 8.809747466353355e-07,
+      "loss": 0.0983,
+      "num_tokens": 10036120.0,
+      "reward": 0.77154541015625,
+      "reward_std": 0.01017143577337265,
+      "rewards//mean": 0.77154541015625,
+      "rewards//std": 0.027987154200673103,
+      "step": 1161
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2324,
+      "grad_norm": 13.446959495544434,
+      "kl": 6.195372574031353,
+      "learning_rate": 8.807691537683684e-07,
+      "loss": 0.2478,
+      "num_tokens": 10044704.0,
+      "reward": 0.7376708984375,
+      "reward_std": 0.013385182246565819,
+      "rewards//mean": 0.7376708984375,
+      "rewards//std": 0.03089313581585884,
+      "step": 1162
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2326,
+      "grad_norm": 11.518983840942383,
+      "kl": 1.5299587156623602,
+      "learning_rate": 8.805634075276317e-07,
+      "loss": 0.0612,
+      "num_tokens": 10053336.0,
+      "reward": 0.7630615234375,
+      "reward_std": 0.006102503277361393,
+      "rewards//mean": 0.7630615234375,
+      "rewards//std": 0.02801506221294403,
+      "step": 1163
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2328,
+      "grad_norm": 3.117992401123047,
+      "kl": 2.9440271724015474,
+      "learning_rate": 8.80357507996e-07,
+      "loss": 0.1178,
+      "num_tokens": 10061936.0,
+      "reward": 0.797119140625,
+      "reward_std": 0.008773138746619225,
+      "rewards//mean": 0.797119140625,
+      "rewards//std": 0.019610146060585976,
+      "step": 1164
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.233,
+      "grad_norm": 3.4370851516723633,
+      "kl": 2.1740373596549034,
+      "learning_rate": 8.801514552564095e-07,
+      "loss": 0.087,
+      "num_tokens": 10070680.0,
+      "reward": 0.78009033203125,
+      "reward_std": 0.007442031055688858,
+      "rewards//mean": 0.78009033203125,
+      "rewards//std": 0.024653080850839615,
+      "step": 1165
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2332,
+      "grad_norm": 27.170595169067383,
+      "kl": 10.168828338384628,
+      "learning_rate": 8.799452493918585e-07,
+      "loss": 0.4068,
+      "num_tokens": 10079240.0,
+      "reward": 0.74139404296875,
+      "reward_std": 0.014152072370052338,
+      "rewards//mean": 0.74139404296875,
+      "rewards//std": 0.03934779018163681,
+      "step": 1166
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2334,
+      "grad_norm": 3.7726423740386963,
+      "kl": 1.264550181105733,
+      "learning_rate": 8.797388904854063e-07,
+      "loss": 0.0506,
+      "num_tokens": 10087896.0,
+      "reward": 0.75830078125,
+      "reward_std": 0.0036158605944365263,
+      "rewards//mean": 0.75830078125,
+      "rewards//std": 0.025614995509386063,
+      "step": 1167
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2336,
+      "grad_norm": 2.8917274475097656,
+      "kl": 2.9061081781983376,
+      "learning_rate": 8.795323786201745e-07,
+      "loss": 0.1162,
+      "num_tokens": 10096480.0,
+      "reward": 0.7794189453125,
+      "reward_std": 0.009156651794910431,
+      "rewards//mean": 0.7794189453125,
+      "rewards//std": 0.03096948191523552,
+      "step": 1168
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2338,
+      "grad_norm": 2.762669086456299,
+      "kl": 2.3426098842173815,
+      "learning_rate": 8.79325713879346e-07,
+      "loss": 0.0937,
+      "num_tokens": 10105048.0,
+      "reward": 0.7904052734375,
+      "reward_std": 0.005530244670808315,
+      "rewards//mean": 0.7904052734375,
+      "rewards//std": 0.01978651061654091,
+      "step": 1169
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.234,
+      "grad_norm": 5.916254997253418,
+      "kl": 6.016498297452927,
+      "learning_rate": 8.791188963461652e-07,
+      "loss": 0.2407,
+      "num_tokens": 10113640.0,
+      "reward": 0.73980712890625,
+      "reward_std": 0.01162217278033495,
+      "rewards//mean": 0.73980712890625,
+      "rewards//std": 0.03229149803519249,
+      "step": 1170
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2342,
+      "grad_norm": 6.179854869842529,
+      "kl": 2.633426643908024,
+      "learning_rate": 8.789119261039384e-07,
+      "loss": 0.1053,
+      "num_tokens": 10122320.0,
+      "reward": 0.781005859375,
+      "reward_std": 0.009721913374960423,
+      "rewards//mean": 0.781005859375,
+      "rewards//std": 0.027347072958946228,
+      "step": 1171
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2344,
+      "grad_norm": 11.493363380432129,
+      "kl": 4.576944898813963,
+      "learning_rate": 8.78704803236033e-07,
+      "loss": 0.1831,
+      "num_tokens": 10131064.0,
+      "reward": 0.72662353515625,
+      "reward_std": 0.013162564486265182,
+      "rewards//mean": 0.72662353515625,
+      "rewards//std": 0.022803641855716705,
+      "step": 1172
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2346,
+      "grad_norm": 3.578338146209717,
+      "kl": 1.9992002733051777,
+      "learning_rate": 8.784975278258782e-07,
+      "loss": 0.08,
+      "num_tokens": 10139696.0,
+      "reward": 0.73089599609375,
+      "reward_std": 0.004772099666297436,
+      "rewards//mean": 0.73089599609375,
+      "rewards//std": 0.03094717301428318,
+      "step": 1173
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2348,
+      "grad_norm": 2.799098491668701,
+      "kl": 0.847711768001318,
+      "learning_rate": 8.782900999569645e-07,
+      "loss": 0.0339,
+      "num_tokens": 10148320.0,
+      "reward": 0.75628662109375,
+      "reward_std": 0.0023677253630012274,
+      "rewards//mean": 0.75628662109375,
+      "rewards//std": 0.022365286946296692,
+      "step": 1174
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.235,
+      "grad_norm": 3.236978769302368,
+      "kl": 4.5400149412453175,
+      "learning_rate": 8.780825197128437e-07,
+      "loss": 0.1816,
+      "num_tokens": 10156992.0,
+      "reward": 0.787353515625,
+      "reward_std": 0.013031869195401669,
+      "rewards//mean": 0.787353515625,
+      "rewards//std": 0.0330202579498291,
+      "step": 1175
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2352,
+      "grad_norm": 6.243363380432129,
+      "kl": 4.9968556091189384,
+      "learning_rate": 8.778747871771291e-07,
+      "loss": 0.1999,
+      "num_tokens": 10165576.0,
+      "reward": 0.7786865234375,
+      "reward_std": 0.014137041755020618,
+      "rewards//mean": 0.7786865234375,
+      "rewards//std": 0.021135948598384857,
+      "step": 1176
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2354,
+      "grad_norm": 5.1518988609313965,
+      "kl": 6.310009470209479,
+      "learning_rate": 8.776669024334955e-07,
+      "loss": 0.2524,
+      "num_tokens": 10174256.0,
+      "reward": 0.76739501953125,
+      "reward_std": 0.01620473712682724,
+      "rewards//mean": 0.76739501953125,
+      "rewards//std": 0.026647798717021942,
+      "step": 1177
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2356,
+      "grad_norm": 7.432278633117676,
+      "kl": 1.9024493284523487,
+      "learning_rate": 8.774588655656787e-07,
+      "loss": 0.0761,
+      "num_tokens": 10182920.0,
+      "reward": 0.72821044921875,
+      "reward_std": 0.005631529726088047,
+      "rewards//mean": 0.72821044921875,
+      "rewards//std": 0.0313568115234375,
+      "step": 1178
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2358,
+      "grad_norm": 3.60760498046875,
+      "kl": 2.6997303441166878,
+      "learning_rate": 8.772506766574761e-07,
+      "loss": 0.108,
+      "num_tokens": 10191600.0,
+      "reward": 0.77288818359375,
+      "reward_std": 0.010570807382464409,
+      "rewards//mean": 0.77288818359375,
+      "rewards//std": 0.025025440379977226,
+      "step": 1179
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.236,
+      "grad_norm": 2.924609422683716,
+      "kl": 2.924817770719528,
+      "learning_rate": 8.770423357927462e-07,
+      "loss": 0.117,
+      "num_tokens": 10200344.0,
+      "reward": 0.73712158203125,
+      "reward_std": 0.00714841578155756,
+      "rewards//mean": 0.73712158203125,
+      "rewards//std": 0.024416156113147736,
+      "step": 1180
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2362,
+      "grad_norm": 4.570566654205322,
+      "kl": 3.667961521074176,
+      "learning_rate": 8.768338430554082e-07,
+      "loss": 0.1467,
+      "num_tokens": 10208960.0,
+      "reward": 0.7537841796875,
+      "reward_std": 0.00938231311738491,
+      "rewards//mean": 0.7537841796875,
+      "rewards//std": 0.039622530341148376,
+      "step": 1181
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2364,
+      "grad_norm": 13.069175720214844,
+      "kl": 5.0878641698509455,
+      "learning_rate": 8.766251985294434e-07,
+      "loss": 0.2035,
+      "num_tokens": 10217648.0,
+      "reward": 0.74945068359375,
+      "reward_std": 0.005997729487717152,
+      "rewards//mean": 0.74945068359375,
+      "rewards//std": 0.025102747604250908,
+      "step": 1182
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2366,
+      "grad_norm": 7.391311168670654,
+      "kl": 2.7894755378365517,
+      "learning_rate": 8.764164022988937e-07,
+      "loss": 0.1116,
+      "num_tokens": 10226272.0,
+      "reward": 0.7518310546875,
+      "reward_std": 0.0049908217042684555,
+      "rewards//mean": 0.7518310546875,
+      "rewards//std": 0.024929456412792206,
+      "step": 1183
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2368,
+      "grad_norm": 12.16387939453125,
+      "kl": 6.9465586617589,
+      "learning_rate": 8.762074544478621e-07,
+      "loss": 0.2779,
+      "num_tokens": 10234856.0,
+      "reward": 0.750244140625,
+      "reward_std": 0.013612458482384682,
+      "rewards//mean": 0.750244140625,
+      "rewards//std": 0.031062457710504532,
+      "step": 1184
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.237,
+      "grad_norm": 5.594395160675049,
+      "kl": 5.4297779724001884,
+      "learning_rate": 8.75998355060513e-07,
+      "loss": 0.2172,
+      "num_tokens": 10243440.0,
+      "reward": 0.73248291015625,
+      "reward_std": 0.010458076372742653,
+      "rewards//mean": 0.73248291015625,
+      "rewards//std": 0.030765656381845474,
+      "step": 1185
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2372,
+      "grad_norm": 3.4841830730438232,
+      "kl": 0.9060949124395847,
+      "learning_rate": 8.757891042210712e-07,
+      "loss": 0.0362,
+      "num_tokens": 10252096.0,
+      "reward": 0.78179931640625,
+      "reward_std": 0.0035348234232515097,
+      "rewards//mean": 0.78179931640625,
+      "rewards//std": 0.016005825251340866,
+      "step": 1186
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2374,
+      "grad_norm": 6.8736252784729,
+      "kl": 4.493726696819067,
+      "learning_rate": 8.755797020138234e-07,
+      "loss": 0.1797,
+      "num_tokens": 10260736.0,
+      "reward": 0.74163818359375,
+      "reward_std": 0.014008762314915657,
+      "rewards//mean": 0.74163818359375,
+      "rewards//std": 0.0290810689330101,
+      "step": 1187
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2376,
+      "grad_norm": 2.9372668266296387,
+      "kl": 0.9298593588173389,
+      "learning_rate": 8.753701485231164e-07,
+      "loss": 0.0372,
+      "num_tokens": 10269320.0,
+      "reward": 0.76458740234375,
+      "reward_std": 0.003342084353789687,
+      "rewards//mean": 0.76458740234375,
+      "rewards//std": 0.03158337250351906,
+      "step": 1188
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2378,
+      "grad_norm": 4.1192498207092285,
+      "kl": 4.80846232175827,
+      "learning_rate": 8.751604438333586e-07,
+      "loss": 0.1923,
+      "num_tokens": 10278008.0,
+      "reward": 0.769775390625,
+      "reward_std": 0.011125459335744381,
+      "rewards//mean": 0.769775390625,
+      "rewards//std": 0.026936586946249008,
+      "step": 1189
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.238,
+      "grad_norm": 5.081353187561035,
+      "kl": 0.7407982870936394,
+      "learning_rate": 8.749505880290188e-07,
+      "loss": 0.0296,
+      "num_tokens": 10286632.0,
+      "reward": 0.7650146484375,
+      "reward_std": 0.002178550697863102,
+      "rewards//mean": 0.7650146484375,
+      "rewards//std": 0.026145800948143005,
+      "step": 1190
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2382,
+      "grad_norm": 5.152669906616211,
+      "kl": 1.9680843688547611,
+      "learning_rate": 8.74740581194627e-07,
+      "loss": 0.0787,
+      "num_tokens": 10295192.0,
+      "reward": 0.75360107421875,
+      "reward_std": 0.008662248030304909,
+      "rewards//mean": 0.75360107421875,
+      "rewards//std": 0.021556738764047623,
+      "step": 1191
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2384,
+      "grad_norm": 4.382171630859375,
+      "kl": 2.6179375406354666,
+      "learning_rate": 8.745304234147739e-07,
+      "loss": 0.1047,
+      "num_tokens": 10303872.0,
+      "reward": 0.7445068359375,
+      "reward_std": 0.01046273298561573,
+      "rewards//mean": 0.7445068359375,
+      "rewards//std": 0.03477679565548897,
+      "step": 1192
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2386,
+      "grad_norm": 6.248373508453369,
+      "kl": 2.0443174801766872,
+      "learning_rate": 8.743201147741111e-07,
+      "loss": 0.0818,
+      "num_tokens": 10312528.0,
+      "reward": 0.75799560546875,
+      "reward_std": 0.009483539499342442,
+      "rewards//mean": 0.75799560546875,
+      "rewards//std": 0.02377026528120041,
+      "step": 1193
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2388,
+      "grad_norm": 3.7448031902313232,
+      "kl": 2.289671115577221,
+      "learning_rate": 8.741096553573506e-07,
+      "loss": 0.0916,
+      "num_tokens": 10321160.0,
+      "reward": 0.75982666015625,
+      "reward_std": 0.005153092555701733,
+      "rewards//mean": 0.75982666015625,
+      "rewards//std": 0.03834277763962746,
+      "step": 1194
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.239,
+      "grad_norm": 6.080252170562744,
+      "kl": 3.648873306810856,
+      "learning_rate": 8.73899045249266e-07,
+      "loss": 0.146,
+      "num_tokens": 10329808.0,
+      "reward": 0.7757568359375,
+      "reward_std": 0.015079613775014877,
+      "rewards//mean": 0.7757568359375,
+      "rewards//std": 0.019409576430916786,
+      "step": 1195
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2392,
+      "grad_norm": 8.778388023376465,
+      "kl": 5.553971689194441,
+      "learning_rate": 8.736882845346905e-07,
+      "loss": 0.2222,
+      "num_tokens": 10338520.0,
+      "reward": 0.72332763671875,
+      "reward_std": 0.010533872991800308,
+      "rewards//mean": 0.72332763671875,
+      "rewards//std": 0.025454264134168625,
+      "step": 1196
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2394,
+      "grad_norm": 4.520699977874756,
+      "kl": 4.532348919659853,
+      "learning_rate": 8.734773732985185e-07,
+      "loss": 0.1813,
+      "num_tokens": 10347160.0,
+      "reward": 0.736328125,
+      "reward_std": 0.010841740295290947,
+      "rewards//mean": 0.736328125,
+      "rewards//std": 0.03413914144039154,
+      "step": 1197
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2396,
+      "grad_norm": 6.137914180755615,
+      "kl": 2.723930537700653,
+      "learning_rate": 8.732663116257055e-07,
+      "loss": 0.109,
+      "num_tokens": 10355760.0,
+      "reward": 0.788818359375,
+      "reward_std": 0.009902745485305786,
+      "rewards//mean": 0.788818359375,
+      "rewards//std": 0.030606873333454132,
+      "step": 1198
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2398,
+      "grad_norm": 5.281278610229492,
+      "kl": 3.864069325849414,
+      "learning_rate": 8.730550996012667e-07,
+      "loss": 0.1546,
+      "num_tokens": 10364384.0,
+      "reward": 0.759521484375,
+      "reward_std": 0.014747949317097664,
+      "rewards//mean": 0.759521484375,
+      "rewards//std": 0.024772604927420616,
+      "step": 1199
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.24,
+      "grad_norm": 3.3508472442626953,
+      "kl": 1.315210809931159,
+      "learning_rate": 8.728437373102784e-07,
+      "loss": 0.0526,
+      "num_tokens": 10372912.0,
+      "reward": 0.7799072265625,
+      "reward_std": 0.005359763279557228,
+      "rewards//mean": 0.7799072265625,
+      "rewards//std": 0.015472638420760632,
+      "step": 1200
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2402,
+      "grad_norm": 19.24559211730957,
+      "kl": 2.4809737019240856,
+      "learning_rate": 8.726322248378774e-07,
+      "loss": 0.0992,
+      "num_tokens": 10381504.0,
+      "reward": 0.7843017578125,
+      "reward_std": 0.010098189115524292,
+      "rewards//mean": 0.7843017578125,
+      "rewards//std": 0.023760473355650902,
+      "step": 1201
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2404,
+      "grad_norm": 7.235874652862549,
+      "kl": 3.858848439529538,
+      "learning_rate": 8.724205622692606e-07,
+      "loss": 0.1544,
+      "num_tokens": 10390168.0,
+      "reward": 0.7529296875,
+      "reward_std": 0.009310659021139145,
+      "rewards//mean": 0.7529296875,
+      "rewards//std": 0.02783101238310337,
+      "step": 1202
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2406,
+      "grad_norm": 10.270509719848633,
+      "kl": 5.628273021429777,
+      "learning_rate": 8.72208749689686e-07,
+      "loss": 0.2251,
+      "num_tokens": 10398736.0,
+      "reward": 0.7620849609375,
+      "reward_std": 0.011618580669164658,
+      "rewards//mean": 0.7620849609375,
+      "rewards//std": 0.021773774176836014,
+      "step": 1203
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2408,
+      "grad_norm": 5.030480861663818,
+      "kl": 3.306698674336076,
+      "learning_rate": 8.719967871844715e-07,
+      "loss": 0.1323,
+      "num_tokens": 10407320.0,
+      "reward": 0.7060546875,
+      "reward_std": 0.010522833094000816,
+      "rewards//mean": 0.7060546875,
+      "rewards//std": 0.034555185586214066,
+      "step": 1204
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.241,
+      "grad_norm": 10.812642097473145,
+      "kl": 3.850353153422475,
+      "learning_rate": 8.717846748389955e-07,
+      "loss": 0.154,
+      "num_tokens": 10415912.0,
+      "reward": 0.7244873046875,
+      "reward_std": 0.008431274443864822,
+      "rewards//mean": 0.7244873046875,
+      "rewards//std": 0.030906852334737778,
+      "step": 1205
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2412,
+      "grad_norm": 27.891611099243164,
+      "kl": 3.9897553957998753,
+      "learning_rate": 8.71572412738697e-07,
+      "loss": 0.1596,
+      "num_tokens": 10424592.0,
+      "reward": 0.7591552734375,
+      "reward_std": 0.011899648234248161,
+      "rewards//mean": 0.7591552734375,
+      "rewards//std": 0.02948923222720623,
+      "step": 1206
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2414,
+      "grad_norm": 10.925292015075684,
+      "kl": 4.316429942846298,
+      "learning_rate": 8.713600009690751e-07,
+      "loss": 0.1727,
+      "num_tokens": 10433208.0,
+      "reward": 0.751953125,
+      "reward_std": 0.005849565379321575,
+      "rewards//mean": 0.751953125,
+      "rewards//std": 0.03435836732387543,
+      "step": 1207
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2416,
+      "grad_norm": 4.999733924865723,
+      "kl": 5.057210177183151,
+      "learning_rate": 8.711474396156892e-07,
+      "loss": 0.2023,
+      "num_tokens": 10441728.0,
+      "reward": 0.74945068359375,
+      "reward_std": 0.010444171726703644,
+      "rewards//mean": 0.74945068359375,
+      "rewards//std": 0.029230598360300064,
+      "step": 1208
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2418,
+      "grad_norm": 9.316264152526855,
+      "kl": 2.889298852533102,
+      "learning_rate": 8.709347287641592e-07,
+      "loss": 0.1156,
+      "num_tokens": 10450336.0,
+      "reward": 0.778076171875,
+      "reward_std": 0.012760389596223831,
+      "rewards//mean": 0.778076171875,
+      "rewards//std": 0.025769364088773727,
+      "step": 1209
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.242,
+      "grad_norm": 12.889250755310059,
+      "kl": 2.478827353566885,
+      "learning_rate": 8.707218685001646e-07,
+      "loss": 0.0992,
+      "num_tokens": 10458856.0,
+      "reward": 0.74981689453125,
+      "reward_std": 0.007895766757428646,
+      "rewards//mean": 0.74981689453125,
+      "rewards//std": 0.025338629260659218,
+      "step": 1210
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2422,
+      "grad_norm": 8.895816802978516,
+      "kl": 4.8706330712884665,
+      "learning_rate": 8.705088589094458e-07,
+      "loss": 0.1948,
+      "num_tokens": 10467424.0,
+      "reward": 0.72357177734375,
+      "reward_std": 0.011275717057287693,
+      "rewards//mean": 0.72357177734375,
+      "rewards//std": 0.037478696554899216,
+      "step": 1211
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2424,
+      "grad_norm": 13.54712200164795,
+      "kl": 8.199546912685037,
+      "learning_rate": 8.702957000778029e-07,
+      "loss": 0.328,
+      "num_tokens": 10476160.0,
+      "reward": 0.7725830078125,
+      "reward_std": 0.019813334569334984,
+      "rewards//mean": 0.7725830078125,
+      "rewards//std": 0.04365084320306778,
+      "step": 1212
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2426,
+      "grad_norm": 4.642307758331299,
+      "kl": 4.662842875346541,
+      "learning_rate": 8.700823920910963e-07,
+      "loss": 0.1865,
+      "num_tokens": 10484800.0,
+      "reward": 0.744384765625,
+      "reward_std": 0.01050968375056982,
+      "rewards//mean": 0.744384765625,
+      "rewards//std": 0.026207387447357178,
+      "step": 1213
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2428,
+      "grad_norm": 7.426133155822754,
+      "kl": 5.004080157727003,
+      "learning_rate": 8.698689350352464e-07,
+      "loss": 0.2002,
+      "num_tokens": 10493448.0,
+      "reward": 0.79730224609375,
+      "reward_std": 0.01092614233493805,
+      "rewards//mean": 0.79730224609375,
+      "rewards//std": 0.021878069266676903,
+      "step": 1214
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.243,
+      "grad_norm": 8.481030464172363,
+      "kl": 3.897951491177082,
+      "learning_rate": 8.696553289962337e-07,
+      "loss": 0.1559,
+      "num_tokens": 10502088.0,
+      "reward": 0.78448486328125,
+      "reward_std": 0.008908973075449467,
+      "rewards//mean": 0.78448486328125,
+      "rewards//std": 0.016856174916028976,
+      "step": 1215
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2432,
+      "grad_norm": 14.525918960571289,
+      "kl": 4.0093574449419975,
+      "learning_rate": 8.694415740600988e-07,
+      "loss": 0.1604,
+      "num_tokens": 10510608.0,
+      "reward": 0.76397705078125,
+      "reward_std": 0.011123032309114933,
+      "rewards//mean": 0.76397705078125,
+      "rewards//std": 0.028158091008663177,
+      "step": 1216
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2434,
+      "grad_norm": 10.636209487915039,
+      "kl": 4.186413774266839,
+      "learning_rate": 8.69227670312942e-07,
+      "loss": 0.1675,
+      "num_tokens": 10519232.0,
+      "reward": 0.767333984375,
+      "reward_std": 0.012185238301753998,
+      "rewards//mean": 0.767333984375,
+      "rewards//std": 0.021051626652479172,
+      "step": 1217
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2436,
+      "grad_norm": 40.315582275390625,
+      "kl": 6.443615116178989,
+      "learning_rate": 8.690136178409235e-07,
+      "loss": 0.2577,
+      "num_tokens": 10527872.0,
+      "reward": 0.74310302734375,
+      "reward_std": 0.015162860043346882,
+      "rewards//mean": 0.74310302734375,
+      "rewards//std": 0.029179802164435387,
+      "step": 1218
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2438,
+      "grad_norm": 10.690977096557617,
+      "kl": 4.274382846429944,
+      "learning_rate": 8.687994167302641e-07,
+      "loss": 0.171,
+      "num_tokens": 10536560.0,
+      "reward": 0.71697998046875,
+      "reward_std": 0.010151637718081474,
+      "rewards//mean": 0.71697998046875,
+      "rewards//std": 0.024635881185531616,
+      "step": 1219
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.244,
+      "grad_norm": 34.73554992675781,
+      "kl": 7.326903743669391,
+      "learning_rate": 8.685850670672438e-07,
+      "loss": 0.2931,
+      "num_tokens": 10545168.0,
+      "reward": 0.7501220703125,
+      "reward_std": 0.010396946221590042,
+      "rewards//mean": 0.7501220703125,
+      "rewards//std": 0.028805769979953766,
+      "step": 1220
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2442,
+      "grad_norm": 34.42318344116211,
+      "kl": 6.078345267102122,
+      "learning_rate": 8.683705689382024e-07,
+      "loss": 0.2431,
+      "num_tokens": 10553912.0,
+      "reward": 0.72015380859375,
+      "reward_std": 0.013924827799201012,
+      "rewards//mean": 0.72015380859375,
+      "rewards//std": 0.02029588632285595,
+      "step": 1221
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2444,
+      "grad_norm": 8.692242622375488,
+      "kl": 3.378894504159689,
+      "learning_rate": 8.6815592242954e-07,
+      "loss": 0.1352,
+      "num_tokens": 10562520.0,
+      "reward": 0.72772216796875,
+      "reward_std": 0.011134006083011627,
+      "rewards//mean": 0.72772216796875,
+      "rewards//std": 0.037645939737558365,
+      "step": 1222
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2446,
+      "grad_norm": 13.238306999206543,
+      "kl": 4.720803817734122,
+      "learning_rate": 8.67941127627716e-07,
+      "loss": 0.1888,
+      "num_tokens": 10571288.0,
+      "reward": 0.77630615234375,
+      "reward_std": 0.01717049442231655,
+      "rewards//mean": 0.77630615234375,
+      "rewards//std": 0.03105752356350422,
+      "step": 1223
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2448,
+      "grad_norm": 25.972370147705078,
+      "kl": 3.125759171321988,
+      "learning_rate": 8.677261846192499e-07,
+      "loss": 0.125,
+      "num_tokens": 10579888.0,
+      "reward": 0.75506591796875,
+      "reward_std": 0.011132306419312954,
+      "rewards//mean": 0.75506591796875,
+      "rewards//std": 0.03039887361228466,
+      "step": 1224
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.245,
+      "grad_norm": 9.226985931396484,
+      "kl": 6.003644030541182,
+      "learning_rate": 8.675110934907204e-07,
+      "loss": 0.2401,
+      "num_tokens": 10588520.0,
+      "reward": 0.791748046875,
+      "reward_std": 0.01744174212217331,
+      "rewards//mean": 0.791748046875,
+      "rewards//std": 0.02488965168595314,
+      "step": 1225
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2452,
+      "grad_norm": 8.872241973876953,
+      "kl": 2.977636018767953,
+      "learning_rate": 8.672958543287666e-07,
+      "loss": 0.1191,
+      "num_tokens": 10597080.0,
+      "reward": 0.77490234375,
+      "reward_std": 0.009975229389965534,
+      "rewards//mean": 0.77490234375,
+      "rewards//std": 0.021793579682707787,
+      "step": 1226
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2454,
+      "grad_norm": 11.018606185913086,
+      "kl": 1.4360513500869274,
+      "learning_rate": 8.670804672200865e-07,
+      "loss": 0.0574,
+      "num_tokens": 10605704.0,
+      "reward": 0.74029541015625,
+      "reward_std": 0.009111039340496063,
+      "rewards//mean": 0.74029541015625,
+      "rewards//std": 0.024190053343772888,
+      "step": 1227
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2456,
+      "grad_norm": 7.508191108703613,
+      "kl": 2.0552911534905434,
+      "learning_rate": 8.668649322514381e-07,
+      "loss": 0.0822,
+      "num_tokens": 10614352.0,
+      "reward": 0.74603271484375,
+      "reward_std": 0.011121081188321114,
+      "rewards//mean": 0.74603271484375,
+      "rewards//std": 0.03067498840391636,
+      "step": 1228
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2458,
+      "grad_norm": 5.576145648956299,
+      "kl": 2.9354402031749487,
+      "learning_rate": 8.666492495096389e-07,
+      "loss": 0.1174,
+      "num_tokens": 10622968.0,
+      "reward": 0.7451171875,
+      "reward_std": 0.013886923901736736,
+      "rewards//mean": 0.7451171875,
+      "rewards//std": 0.04022333025932312,
+      "step": 1229
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.246,
+      "grad_norm": 8.805665969848633,
+      "kl": 2.536726539954543,
+      "learning_rate": 8.664334190815659e-07,
+      "loss": 0.1015,
+      "num_tokens": 10631624.0,
+      "reward": 0.74835205078125,
+      "reward_std": 0.0117610152810812,
+      "rewards//mean": 0.74835205078125,
+      "rewards//std": 0.028054682537913322,
+      "step": 1230
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2462,
+      "grad_norm": 7.829744815826416,
+      "kl": 3.846081856638193,
+      "learning_rate": 8.662174410541554e-07,
+      "loss": 0.1538,
+      "num_tokens": 10640224.0,
+      "reward": 0.737548828125,
+      "reward_std": 0.011144118383526802,
+      "rewards//mean": 0.737548828125,
+      "rewards//std": 0.027942834421992302,
+      "step": 1231
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2464,
+      "grad_norm": 6.520112991333008,
+      "kl": 2.071901887655258,
+      "learning_rate": 8.660013155144035e-07,
+      "loss": 0.0829,
+      "num_tokens": 10648752.0,
+      "reward": 0.73602294921875,
+      "reward_std": 0.008494130335748196,
+      "rewards//mean": 0.73602294921875,
+      "rewards//std": 0.023976976051926613,
+      "step": 1232
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2466,
+      "grad_norm": 7.9967875480651855,
+      "kl": 6.293617935851216,
+      "learning_rate": 8.657850425493654e-07,
+      "loss": 0.2517,
+      "num_tokens": 10657368.0,
+      "reward": 0.73828125,
+      "reward_std": 0.02175438404083252,
+      "rewards//mean": 0.73828125,
+      "rewards//std": 0.04595661163330078,
+      "step": 1233
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2468,
+      "grad_norm": 3.7229974269866943,
+      "kl": 3.049281246960163,
+      "learning_rate": 8.65568622246156e-07,
+      "loss": 0.122,
+      "num_tokens": 10666032.0,
+      "reward": 0.77191162109375,
+      "reward_std": 0.008106382563710213,
+      "rewards//mean": 0.77191162109375,
+      "rewards//std": 0.023944754153490067,
+      "step": 1234
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.247,
+      "grad_norm": 14.904300689697266,
+      "kl": 2.1938663106411695,
+      "learning_rate": 8.653520546919493e-07,
+      "loss": 0.0878,
+      "num_tokens": 10674592.0,
+      "reward": 0.79669189453125,
+      "reward_std": 0.011995483189821243,
+      "rewards//mean": 0.79669189453125,
+      "rewards//std": 0.033562660217285156,
+      "step": 1235
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2472,
+      "grad_norm": 9.752826690673828,
+      "kl": 5.046719258651137,
+      "learning_rate": 8.651353399739787e-07,
+      "loss": 0.2019,
+      "num_tokens": 10683232.0,
+      "reward": 0.73748779296875,
+      "reward_std": 0.014928562566637993,
+      "rewards//mean": 0.73748779296875,
+      "rewards//std": 0.0304267480969429,
+      "step": 1236
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2474,
+      "grad_norm": 9.397666931152344,
+      "kl": 5.7591456193476915,
+      "learning_rate": 8.649184781795367e-07,
+      "loss": 0.2304,
+      "num_tokens": 10691880.0,
+      "reward": 0.7884521484375,
+      "reward_std": 0.021809879690408707,
+      "rewards//mean": 0.7884521484375,
+      "rewards//std": 0.03189605847001076,
+      "step": 1237
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2476,
+      "grad_norm": 15.128095626831055,
+      "kl": 6.171701226383448,
+      "learning_rate": 8.647014693959753e-07,
+      "loss": 0.2469,
+      "num_tokens": 10700504.0,
+      "reward": 0.721435546875,
+      "reward_std": 0.010850808583199978,
+      "rewards//mean": 0.721435546875,
+      "rewards//std": 0.033810317516326904,
+      "step": 1238
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2478,
+      "grad_norm": 6.98085880279541,
+      "kl": 2.490553168579936,
+      "learning_rate": 8.644843137107057e-07,
+      "loss": 0.0996,
+      "num_tokens": 10709176.0,
+      "reward": 0.77630615234375,
+      "reward_std": 0.013876741752028465,
+      "rewards//mean": 0.77630615234375,
+      "rewards//std": 0.02504841610789299,
+      "step": 1239
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.248,
+      "grad_norm": 23.153583526611328,
+      "kl": 1.9768635164946318,
+      "learning_rate": 8.642670112111981e-07,
+      "loss": 0.0791,
+      "num_tokens": 10717912.0,
+      "reward": 0.77349853515625,
+      "reward_std": 0.007938895374536514,
+      "rewards//mean": 0.77349853515625,
+      "rewards//std": 0.030109168961644173,
+      "step": 1240
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2482,
+      "grad_norm": 9.379088401794434,
+      "kl": 6.8707516342401505,
+      "learning_rate": 8.64049561984982e-07,
+      "loss": 0.2748,
+      "num_tokens": 10726616.0,
+      "reward": 0.75604248046875,
+      "reward_std": 0.015093580819666386,
+      "rewards//mean": 0.75604248046875,
+      "rewards//std": 0.02899818681180477,
+      "step": 1241
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2484,
+      "grad_norm": 8.29955005645752,
+      "kl": 3.944977167993784,
+      "learning_rate": 8.638319661196459e-07,
+      "loss": 0.1578,
+      "num_tokens": 10735200.0,
+      "reward": 0.76593017578125,
+      "reward_std": 0.007996254600584507,
+      "rewards//mean": 0.76593017578125,
+      "rewards//std": 0.023951709270477295,
+      "step": 1242
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2486,
+      "grad_norm": 31.44389533996582,
+      "kl": 7.717126229777932,
+      "learning_rate": 8.636142237028372e-07,
+      "loss": 0.3087,
+      "num_tokens": 10743880.0,
+      "reward": 0.7412109375,
+      "reward_std": 0.014628959819674492,
+      "rewards//mean": 0.7412109375,
+      "rewards//std": 0.03663065284490585,
+      "step": 1243
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2488,
+      "grad_norm": 31.918787002563477,
+      "kl": 9.728628190234303,
+      "learning_rate": 8.633963348222628e-07,
+      "loss": 0.3891,
+      "num_tokens": 10752560.0,
+      "reward": 0.73858642578125,
+      "reward_std": 0.008714969269931316,
+      "rewards//mean": 0.73858642578125,
+      "rewards//std": 0.03782325237989426,
+      "step": 1244
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.249,
+      "grad_norm": 9.533246040344238,
+      "kl": 2.8249090276658535,
+      "learning_rate": 8.631782995656882e-07,
+      "loss": 0.113,
+      "num_tokens": 10761192.0,
+      "reward": 0.7611083984375,
+      "reward_std": 0.006418757606297731,
+      "rewards//mean": 0.7611083984375,
+      "rewards//std": 0.022522857412695885,
+      "step": 1245
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2492,
+      "grad_norm": 34.13203811645508,
+      "kl": 11.04344275034964,
+      "learning_rate": 8.62960118020938e-07,
+      "loss": 0.4417,
+      "num_tokens": 10769904.0,
+      "reward": 0.7117919921875,
+      "reward_std": 0.010154195129871368,
+      "rewards//mean": 0.7117919921875,
+      "rewards//std": 0.04697548598051071,
+      "step": 1246
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2494,
+      "grad_norm": 16.348169326782227,
+      "kl": 4.125708527863026,
+      "learning_rate": 8.627417902758956e-07,
+      "loss": 0.165,
+      "num_tokens": 10778632.0,
+      "reward": 0.79022216796875,
+      "reward_std": 0.013444384559988976,
+      "rewards//mean": 0.79022216796875,
+      "rewards//std": 0.024926647543907166,
+      "step": 1247
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2496,
+      "grad_norm": 4.244007110595703,
+      "kl": 2.4565203841775656,
+      "learning_rate": 8.625233164185034e-07,
+      "loss": 0.0983,
+      "num_tokens": 10787216.0,
+      "reward": 0.763916015625,
+      "reward_std": 0.009631536900997162,
+      "rewards//mean": 0.763916015625,
+      "rewards//std": 0.0243386123329401,
+      "step": 1248
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2498,
+      "grad_norm": 3.8231735229492188,
+      "kl": 2.839028410613537,
+      "learning_rate": 8.623046965367628e-07,
+      "loss": 0.1136,
+      "num_tokens": 10795792.0,
+      "reward": 0.74566650390625,
+      "reward_std": 0.0094450693577528,
+      "rewards//mean": 0.74566650390625,
+      "rewards//std": 0.02701956033706665,
+      "step": 1249
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.25,
+      "grad_norm": 9.10566234588623,
+      "kl": 6.971977606415749,
+      "learning_rate": 8.620859307187338e-07,
+      "loss": 0.2789,
+      "num_tokens": 10804400.0,
+      "reward": 0.7451171875,
+      "reward_std": 0.014911534264683723,
+      "rewards//mean": 0.7451171875,
+      "rewards//std": 0.030544491484761238,
+      "step": 1250
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2502,
+      "grad_norm": 20.85613250732422,
+      "kl": 8.940727328881621,
+      "learning_rate": 8.61867019052535e-07,
+      "loss": 0.3576,
+      "num_tokens": 10813208.0,
+      "reward": 0.7559814453125,
+      "reward_std": 0.011848023161292076,
+      "rewards//mean": 0.7559814453125,
+      "rewards//std": 0.033167317509651184,
+      "step": 1251
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2504,
+      "grad_norm": 13.552748680114746,
+      "kl": 4.335039634257555,
+      "learning_rate": 8.616479616263444e-07,
+      "loss": 0.1734,
+      "num_tokens": 10821848.0,
+      "reward": 0.7596435546875,
+      "reward_std": 0.01608794741332531,
+      "rewards//mean": 0.7596435546875,
+      "rewards//std": 0.02570437081158161,
+      "step": 1252
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2506,
+      "grad_norm": 2.9355878829956055,
+      "kl": 3.873683985322714,
+      "learning_rate": 8.61428758528398e-07,
+      "loss": 0.1549,
+      "num_tokens": 10830376.0,
+      "reward": 0.7449951171875,
+      "reward_std": 0.007550119888037443,
+      "rewards//mean": 0.7449951171875,
+      "rewards//std": 0.017791688442230225,
+      "step": 1253
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2508,
+      "grad_norm": 4.576368808746338,
+      "kl": 5.775039687752724,
+      "learning_rate": 8.612094098469909e-07,
+      "loss": 0.231,
+      "num_tokens": 10838992.0,
+      "reward": 0.76055908203125,
+      "reward_std": 0.017393331974744797,
+      "rewards//mean": 0.76055908203125,
+      "rewards//std": 0.026177842170000076,
+      "step": 1254
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.251,
+      "grad_norm": 11.842081069946289,
+      "kl": 7.278344709426165,
+      "learning_rate": 8.609899156704767e-07,
+      "loss": 0.2911,
+      "num_tokens": 10847728.0,
+      "reward": 0.7464599609375,
+      "reward_std": 0.010564940050244331,
+      "rewards//mean": 0.7464599609375,
+      "rewards//std": 0.03639662638306618,
+      "step": 1255
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2512,
+      "grad_norm": 6.426313877105713,
+      "kl": 5.643575828522444,
+      "learning_rate": 8.607702760872677e-07,
+      "loss": 0.2257,
+      "num_tokens": 10856544.0,
+      "reward": 0.781005859375,
+      "reward_std": 0.012030715122818947,
+      "rewards//mean": 0.781005859375,
+      "rewards//std": 0.04051655903458595,
+      "step": 1256
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2514,
+      "grad_norm": 3.890913724899292,
+      "kl": 3.2777351792901754,
+      "learning_rate": 8.605504911858346e-07,
+      "loss": 0.1311,
+      "num_tokens": 10865104.0,
+      "reward": 0.750244140625,
+      "reward_std": 0.0073437620885670185,
+      "rewards//mean": 0.750244140625,
+      "rewards//std": 0.022807708010077477,
+      "step": 1257
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2516,
+      "grad_norm": 8.680928230285645,
+      "kl": 1.990102307870984,
+      "learning_rate": 8.603305610547069e-07,
+      "loss": 0.0796,
+      "num_tokens": 10873816.0,
+      "reward": 0.75885009765625,
+      "reward_std": 0.004397744778543711,
+      "rewards//mean": 0.75885009765625,
+      "rewards//std": 0.029266826808452606,
+      "step": 1258
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2518,
+      "grad_norm": 4.191249370574951,
+      "kl": 5.214279746636748,
+      "learning_rate": 8.601104857824722e-07,
+      "loss": 0.2086,
+      "num_tokens": 10882440.0,
+      "reward": 0.78961181640625,
+      "reward_std": 0.014969921670854092,
+      "rewards//mean": 0.78961181640625,
+      "rewards//std": 0.03335590288043022,
+      "step": 1259
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.252,
+      "grad_norm": 8.555057525634766,
+      "kl": 3.9011244364082813,
+      "learning_rate": 8.598902654577768e-07,
+      "loss": 0.156,
+      "num_tokens": 10891048.0,
+      "reward": 0.7681884765625,
+      "reward_std": 0.019131936132907867,
+      "rewards//mean": 0.7681884765625,
+      "rewards//std": 0.030466832220554352,
+      "step": 1260
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2522,
+      "grad_norm": 5.7329816818237305,
+      "kl": 4.792279714718461,
+      "learning_rate": 8.596699001693255e-07,
+      "loss": 0.1917,
+      "num_tokens": 10899720.0,
+      "reward": 0.7528076171875,
+      "reward_std": 0.012098362669348717,
+      "rewards//mean": 0.7528076171875,
+      "rewards//std": 0.032245345413684845,
+      "step": 1261
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2524,
+      "grad_norm": 7.551224708557129,
+      "kl": 3.247972499579191,
+      "learning_rate": 8.594493900058816e-07,
+      "loss": 0.1299,
+      "num_tokens": 10908304.0,
+      "reward": 0.77276611328125,
+      "reward_std": 0.015282051637768745,
+      "rewards//mean": 0.77276611328125,
+      "rewards//std": 0.04180915653705597,
+      "step": 1262
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2526,
+      "grad_norm": 5.042646884918213,
+      "kl": 5.161837503314018,
+      "learning_rate": 8.592287350562663e-07,
+      "loss": 0.2065,
+      "num_tokens": 10916880.0,
+      "reward": 0.79107666015625,
+      "reward_std": 0.015731535851955414,
+      "rewards//mean": 0.79107666015625,
+      "rewards//std": 0.034597158432006836,
+      "step": 1263
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2528,
+      "grad_norm": 14.095562934875488,
+      "kl": 1.6300823576748371,
+      "learning_rate": 8.590079354093593e-07,
+      "loss": 0.0652,
+      "num_tokens": 10925520.0,
+      "reward": 0.77783203125,
+      "reward_std": 0.0067242844961583614,
+      "rewards//mean": 0.77783203125,
+      "rewards//std": 0.022579578682780266,
+      "step": 1264
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.253,
+      "grad_norm": 3.9633383750915527,
+      "kl": 1.2054816894233227,
+      "learning_rate": 8.587869911540992e-07,
+      "loss": 0.0482,
+      "num_tokens": 10934168.0,
+      "reward": 0.7489013671875,
+      "reward_std": 0.004060286097228527,
+      "rewards//mean": 0.7489013671875,
+      "rewards//std": 0.025358112528920174,
+      "step": 1265
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2532,
+      "grad_norm": 4.64142370223999,
+      "kl": 3.954777590930462,
+      "learning_rate": 8.585659023794818e-07,
+      "loss": 0.1582,
+      "num_tokens": 10942832.0,
+      "reward": 0.7490234375,
+      "reward_std": 0.013307355344295502,
+      "rewards//mean": 0.7490234375,
+      "rewards//std": 0.034195851534605026,
+      "step": 1266
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2534,
+      "grad_norm": 7.686950206756592,
+      "kl": 3.5369584299623966,
+      "learning_rate": 8.583446691745617e-07,
+      "loss": 0.1415,
+      "num_tokens": 10951568.0,
+      "reward": 0.784912109375,
+      "reward_std": 0.01466282270848751,
+      "rewards//mean": 0.784912109375,
+      "rewards//std": 0.026198145002126694,
+      "step": 1267
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2536,
+      "grad_norm": 4.102766990661621,
+      "kl": 4.478368993848562,
+      "learning_rate": 8.581232916284517e-07,
+      "loss": 0.1791,
+      "num_tokens": 10960248.0,
+      "reward": 0.75958251953125,
+      "reward_std": 0.00903963390737772,
+      "rewards//mean": 0.75958251953125,
+      "rewards//std": 0.02239031344652176,
+      "step": 1268
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2538,
+      "grad_norm": 5.248315811157227,
+      "kl": 2.6819275487214327,
+      "learning_rate": 8.579017698303228e-07,
+      "loss": 0.1073,
+      "num_tokens": 10968904.0,
+      "reward": 0.75897216796875,
+      "reward_std": 0.009780505672097206,
+      "rewards//mean": 0.75897216796875,
+      "rewards//std": 0.032768476754426956,
+      "step": 1269
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.254,
+      "grad_norm": 4.820333003997803,
+      "kl": 2.1646746415644884,
+      "learning_rate": 8.576801038694039e-07,
+      "loss": 0.0866,
+      "num_tokens": 10977560.0,
+      "reward": 0.76324462890625,
+      "reward_std": 0.00664786109700799,
+      "rewards//mean": 0.76324462890625,
+      "rewards//std": 0.027353644371032715,
+      "step": 1270
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2542,
+      "grad_norm": 7.026268482208252,
+      "kl": 3.008241306990385,
+      "learning_rate": 8.574582938349817e-07,
+      "loss": 0.1203,
+      "num_tokens": 10986264.0,
+      "reward": 0.77130126953125,
+      "reward_std": 0.01176033541560173,
+      "rewards//mean": 0.77130126953125,
+      "rewards//std": 0.026133855804800987,
+      "step": 1271
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2544,
+      "grad_norm": 13.753020286560059,
+      "kl": 1.1150043960660696,
+      "learning_rate": 8.572363398164016e-07,
+      "loss": 0.0446,
+      "num_tokens": 10994872.0,
+      "reward": 0.75775146484375,
+      "reward_std": 0.0015412584180012345,
+      "rewards//mean": 0.75775146484375,
+      "rewards//std": 0.019015923142433167,
+      "step": 1272
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2546,
+      "grad_norm": 6.849703311920166,
+      "kl": 1.1169192641973495,
+      "learning_rate": 8.570142419030666e-07,
+      "loss": 0.0447,
+      "num_tokens": 11003560.0,
+      "reward": 0.7681884765625,
+      "reward_std": 0.0015413069631904364,
+      "rewards//mean": 0.7681884765625,
+      "rewards//std": 0.02633042074739933,
+      "step": 1273
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2548,
+      "grad_norm": 2.7238874435424805,
+      "kl": 2.621772589161992,
+      "learning_rate": 8.567920001844375e-07,
+      "loss": 0.1049,
+      "num_tokens": 11012160.0,
+      "reward": 0.76214599609375,
+      "reward_std": 0.007980536669492722,
+      "rewards//mean": 0.76214599609375,
+      "rewards//std": 0.02573402039706707,
+      "step": 1274
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.255,
+      "grad_norm": 12.22774887084961,
+      "kl": 4.507930127903819,
+      "learning_rate": 8.565696147500337e-07,
+      "loss": 0.1803,
+      "num_tokens": 11020816.0,
+      "reward": 0.75836181640625,
+      "reward_std": 0.010539361275732517,
+      "rewards//mean": 0.75836181640625,
+      "rewards//std": 0.0307173989713192,
+      "step": 1275
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2552,
+      "grad_norm": 7.131739139556885,
+      "kl": 3.0699077248573303,
+      "learning_rate": 8.563470856894314e-07,
+      "loss": 0.1228,
+      "num_tokens": 11029440.0,
+      "reward": 0.76611328125,
+      "reward_std": 0.008369958959519863,
+      "rewards//mean": 0.76611328125,
+      "rewards//std": 0.01891385391354561,
+      "step": 1276
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2554,
+      "grad_norm": 4.130187511444092,
+      "kl": 4.302004503086209,
+      "learning_rate": 8.561244130922657e-07,
+      "loss": 0.1721,
+      "num_tokens": 11038104.0,
+      "reward": 0.7452392578125,
+      "reward_std": 0.013671613298356533,
+      "rewards//mean": 0.7452392578125,
+      "rewards//std": 0.034474264830350876,
+      "step": 1277
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2556,
+      "grad_norm": 6.271708965301514,
+      "kl": 1.6784346252679825,
+      "learning_rate": 8.559015970482291e-07,
+      "loss": 0.0671,
+      "num_tokens": 11046760.0,
+      "reward": 0.7467041015625,
+      "reward_std": 0.0062317755073308945,
+      "rewards//mean": 0.7467041015625,
+      "rewards//std": 0.03089313581585884,
+      "step": 1278
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2558,
+      "grad_norm": 27.68781089782715,
+      "kl": 4.338345795869827,
+      "learning_rate": 8.556786376470716e-07,
+      "loss": 0.1735,
+      "num_tokens": 11055496.0,
+      "reward": 0.77911376953125,
+      "reward_std": 0.01089534629136324,
+      "rewards//mean": 0.77911376953125,
+      "rewards//std": 0.03209824860095978,
+      "step": 1279
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.256,
+      "grad_norm": 8.381805419921875,
+      "kl": 4.739690890535712,
+      "learning_rate": 8.554555349786015e-07,
+      "loss": 0.1896,
+      "num_tokens": 11064112.0,
+      "reward": 0.7952880859375,
+      "reward_std": 0.013847950845956802,
+      "rewards//mean": 0.7952880859375,
+      "rewards//std": 0.031069036573171616,
+      "step": 1280
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2562,
+      "grad_norm": 3.666189193725586,
+      "kl": 1.6756311561912298,
+      "learning_rate": 8.552322891326844e-07,
+      "loss": 0.067,
+      "num_tokens": 11072752.0,
+      "reward": 0.77252197265625,
+      "reward_std": 0.004807932768017054,
+      "rewards//mean": 0.77252197265625,
+      "rewards//std": 0.010433909483253956,
+      "step": 1281
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2564,
+      "grad_norm": 4.122167110443115,
+      "kl": 2.7426191214472055,
+      "learning_rate": 8.550089001992437e-07,
+      "loss": 0.1097,
+      "num_tokens": 11081424.0,
+      "reward": 0.73468017578125,
+      "reward_std": 0.008176295086741447,
+      "rewards//mean": 0.73468017578125,
+      "rewards//std": 0.036652494221925735,
+      "step": 1282
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2566,
+      "grad_norm": 4.487239837646484,
+      "kl": 3.345269527286291,
+      "learning_rate": 8.547853682682604e-07,
+      "loss": 0.1338,
+      "num_tokens": 11089952.0,
+      "reward": 0.77093505859375,
+      "reward_std": 0.007087345235049725,
+      "rewards//mean": 0.77093505859375,
+      "rewards//std": 0.01962856762111187,
+      "step": 1283
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2568,
+      "grad_norm": 7.786471366882324,
+      "kl": 3.652106024324894,
+      "learning_rate": 8.545616934297733e-07,
+      "loss": 0.1461,
+      "num_tokens": 11098544.0,
+      "reward": 0.79718017578125,
+      "reward_std": 0.016802387312054634,
+      "rewards//mean": 0.79718017578125,
+      "rewards//std": 0.026940450072288513,
+      "step": 1284
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.257,
+      "grad_norm": 5.001603126525879,
+      "kl": 1.5137909315526485,
+      "learning_rate": 8.543378757738784e-07,
+      "loss": 0.0606,
+      "num_tokens": 11107168.0,
+      "reward": 0.7647705078125,
+      "reward_std": 0.0034436597488820553,
+      "rewards//mean": 0.7647705078125,
+      "rewards//std": 0.034319356083869934,
+      "step": 1285
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2572,
+      "grad_norm": 5.244659900665283,
+      "kl": 2.4232273045927286,
+      "learning_rate": 8.541139153907295e-07,
+      "loss": 0.0969,
+      "num_tokens": 11115752.0,
+      "reward": 0.75677490234375,
+      "reward_std": 0.006669648922979832,
+      "rewards//mean": 0.75677490234375,
+      "rewards//std": 0.02266247384250164,
+      "step": 1286
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2574,
+      "grad_norm": 8.395547866821289,
+      "kl": 3.235109167173505,
+      "learning_rate": 8.538898123705379e-07,
+      "loss": 0.1294,
+      "num_tokens": 11124360.0,
+      "reward": 0.78277587890625,
+      "reward_std": 0.008708701469004154,
+      "rewards//mean": 0.78277587890625,
+      "rewards//std": 0.02795793116092682,
+      "step": 1287
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2576,
+      "grad_norm": 1.7316111326217651,
+      "kl": 1.5258556101471186,
+      "learning_rate": 8.536655668035721e-07,
+      "loss": 0.061,
+      "num_tokens": 11133016.0,
+      "reward": 0.767822265625,
+      "reward_std": 0.004365556873381138,
+      "rewards//mean": 0.767822265625,
+      "rewards//std": 0.028398511931300163,
+      "step": 1288
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2578,
+      "grad_norm": 6.039356708526611,
+      "kl": 7.7991479728370905,
+      "learning_rate": 8.534411787801586e-07,
+      "loss": 0.312,
+      "num_tokens": 11141656.0,
+      "reward": 0.771240234375,
+      "reward_std": 0.01720530167222023,
+      "rewards//mean": 0.771240234375,
+      "rewards//std": 0.03166480362415314,
+      "step": 1289
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.258,
+      "grad_norm": 5.303319931030273,
+      "kl": 4.693038806319237,
+      "learning_rate": 8.532166483906802e-07,
+      "loss": 0.1877,
+      "num_tokens": 11150280.0,
+      "reward": 0.7457275390625,
+      "reward_std": 0.008638778701424599,
+      "rewards//mean": 0.7457275390625,
+      "rewards//std": 0.024800391867756844,
+      "step": 1290
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2582,
+      "grad_norm": 3.814932346343994,
+      "kl": 5.850923229008913,
+      "learning_rate": 8.529919757255781e-07,
+      "loss": 0.234,
+      "num_tokens": 11158864.0,
+      "reward": 0.76611328125,
+      "reward_std": 0.02036258764564991,
+      "rewards//mean": 0.76611328125,
+      "rewards//std": 0.03756749629974365,
+      "step": 1291
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2584,
+      "grad_norm": 4.260945796966553,
+      "kl": 5.5106862504035234,
+      "learning_rate": 8.527671608753506e-07,
+      "loss": 0.2204,
+      "num_tokens": 11167488.0,
+      "reward": 0.76123046875,
+      "reward_std": 0.013373324647545815,
+      "rewards//mean": 0.76123046875,
+      "rewards//std": 0.028969526290893555,
+      "step": 1292
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2586,
+      "grad_norm": 5.248500347137451,
+      "kl": 3.916138354688883,
+      "learning_rate": 8.525422039305528e-07,
+      "loss": 0.1566,
+      "num_tokens": 11176208.0,
+      "reward": 0.7728271484375,
+      "reward_std": 0.01021310780197382,
+      "rewards//mean": 0.7728271484375,
+      "rewards//std": 0.027065252885222435,
+      "step": 1293
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2588,
+      "grad_norm": 7.21472692489624,
+      "kl": 5.753823101520538,
+      "learning_rate": 8.523171049817973e-07,
+      "loss": 0.2302,
+      "num_tokens": 11184800.0,
+      "reward": 0.776123046875,
+      "reward_std": 0.011476203799247742,
+      "rewards//mean": 0.776123046875,
+      "rewards//std": 0.026565372943878174,
+      "step": 1294
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.259,
+      "grad_norm": 2.359610080718994,
+      "kl": 3.251587826758623,
+      "learning_rate": 8.520918641197541e-07,
+      "loss": 0.1301,
+      "num_tokens": 11193456.0,
+      "reward": 0.76739501953125,
+      "reward_std": 0.008114458061754704,
+      "rewards//mean": 0.76739501953125,
+      "rewards//std": 0.03128044307231903,
+      "step": 1295
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2592,
+      "grad_norm": 2.9881999492645264,
+      "kl": 2.696640431880951,
+      "learning_rate": 8.518664814351502e-07,
+      "loss": 0.1079,
+      "num_tokens": 11202144.0,
+      "reward": 0.739990234375,
+      "reward_std": 0.005112895276397467,
+      "rewards//mean": 0.739990234375,
+      "rewards//std": 0.02935468964278698,
+      "step": 1296
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2594,
+      "grad_norm": 4.5930891036987305,
+      "kl": 6.463214572519064,
+      "learning_rate": 8.516409570187696e-07,
+      "loss": 0.2585,
+      "num_tokens": 11210904.0,
+      "reward": 0.75799560546875,
+      "reward_std": 0.016281146556138992,
+      "rewards//mean": 0.75799560546875,
+      "rewards//std": 0.036241315305233,
+      "step": 1297
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2596,
+      "grad_norm": 8.910324096679688,
+      "kl": 6.601942088454962,
+      "learning_rate": 8.514152909614535e-07,
+      "loss": 0.2641,
+      "num_tokens": 11219616.0,
+      "reward": 0.76300048828125,
+      "reward_std": 0.015175841748714447,
+      "rewards//mean": 0.76300048828125,
+      "rewards//std": 0.03656483441591263,
+      "step": 1298
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2598,
+      "grad_norm": 3.1343729496002197,
+      "kl": 2.7468247059732676,
+      "learning_rate": 8.511894833541005e-07,
+      "loss": 0.1099,
+      "num_tokens": 11228280.0,
+      "reward": 0.750732421875,
+      "reward_std": 0.004793817177414894,
+      "rewards//mean": 0.750732421875,
+      "rewards//std": 0.021842146292328835,
+      "step": 1299
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.26,
+      "grad_norm": 4.297388076782227,
+      "kl": 2.919129230082035,
+      "learning_rate": 8.509635342876654e-07,
+      "loss": 0.1168,
+      "num_tokens": 11236848.0,
+      "reward": 0.7559814453125,
+      "reward_std": 0.0069838822819292545,
+      "rewards//mean": 0.7559814453125,
+      "rewards//std": 0.028125077486038208,
+      "step": 1300
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2602,
+      "grad_norm": 5.890142917633057,
+      "kl": 4.626889752224088,
+      "learning_rate": 8.507374438531606e-07,
+      "loss": 0.1851,
+      "num_tokens": 11245440.0,
+      "reward": 0.77301025390625,
+      "reward_std": 0.013959040865302086,
+      "rewards//mean": 0.77301025390625,
+      "rewards//std": 0.03568018972873688,
+      "step": 1301
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2604,
+      "grad_norm": 7.185187339782715,
+      "kl": 3.8990981429815292,
+      "learning_rate": 8.505112121416553e-07,
+      "loss": 0.156,
+      "num_tokens": 11254128.0,
+      "reward": 0.733154296875,
+      "reward_std": 0.004352999851107597,
+      "rewards//mean": 0.733154296875,
+      "rewards//std": 0.01750909723341465,
+      "step": 1302
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2606,
+      "grad_norm": 6.5421552658081055,
+      "kl": 3.2562216632068157,
+      "learning_rate": 8.502848392442758e-07,
+      "loss": 0.1302,
+      "num_tokens": 11262768.0,
+      "reward": 0.78045654296875,
+      "reward_std": 0.010951787233352661,
+      "rewards//mean": 0.78045654296875,
+      "rewards//std": 0.034168779850006104,
+      "step": 1303
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2608,
+      "grad_norm": 4.482645034790039,
+      "kl": 4.552934428676963,
+      "learning_rate": 8.500583252522052e-07,
+      "loss": 0.1821,
+      "num_tokens": 11271360.0,
+      "reward": 0.76275634765625,
+      "reward_std": 0.008597548119723797,
+      "rewards//mean": 0.76275634765625,
+      "rewards//std": 0.035531386733055115,
+      "step": 1304
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.261,
+      "grad_norm": 6.0340962409973145,
+      "kl": 3.4215122647583485,
+      "learning_rate": 8.498316702566826e-07,
+      "loss": 0.1369,
+      "num_tokens": 11279992.0,
+      "reward": 0.76104736328125,
+      "reward_std": 0.006469708867371082,
+      "rewards//mean": 0.76104736328125,
+      "rewards//std": 0.015145717188715935,
+      "step": 1305
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2612,
+      "grad_norm": 5.419297218322754,
+      "kl": 2.951256576925516,
+      "learning_rate": 8.496048743490053e-07,
+      "loss": 0.1181,
+      "num_tokens": 11288736.0,
+      "reward": 0.76922607421875,
+      "reward_std": 0.009774585254490376,
+      "rewards//mean": 0.76922607421875,
+      "rewards//std": 0.029250789433717728,
+      "step": 1306
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2614,
+      "grad_norm": 9.354080200195312,
+      "kl": 1.5646381042897701,
+      "learning_rate": 8.493779376205264e-07,
+      "loss": 0.0626,
+      "num_tokens": 11297368.0,
+      "reward": 0.79656982421875,
+      "reward_std": 0.010128369554877281,
+      "rewards//mean": 0.79656982421875,
+      "rewards//std": 0.026063093915581703,
+      "step": 1307
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2616,
+      "grad_norm": 2.8813416957855225,
+      "kl": 1.8132019937038422,
+      "learning_rate": 8.491508601626561e-07,
+      "loss": 0.0725,
+      "num_tokens": 11306032.0,
+      "reward": 0.75103759765625,
+      "reward_std": 0.005398713517934084,
+      "rewards//mean": 0.75103759765625,
+      "rewards//std": 0.03246075287461281,
+      "step": 1308
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2618,
+      "grad_norm": 9.517343521118164,
+      "kl": 3.6751442085951567,
+      "learning_rate": 8.489236420668608e-07,
+      "loss": 0.147,
+      "num_tokens": 11314848.0,
+      "reward": 0.7562255859375,
+      "reward_std": 0.016644582152366638,
+      "rewards//mean": 0.7562255859375,
+      "rewards//std": 0.02966529130935669,
+      "step": 1309
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.262,
+      "grad_norm": 14.530468940734863,
+      "kl": 3.6663004029542208,
+      "learning_rate": 8.486962834246645e-07,
+      "loss": 0.1467,
+      "num_tokens": 11323456.0,
+      "reward": 0.74462890625,
+      "reward_std": 0.004980337806046009,
+      "rewards//mean": 0.74462890625,
+      "rewards//std": 0.028112424537539482,
+      "step": 1310
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2622,
+      "grad_norm": 7.726293563842773,
+      "kl": 2.5143831074237823,
+      "learning_rate": 8.484687843276468e-07,
+      "loss": 0.1006,
+      "num_tokens": 11332048.0,
+      "reward": 0.75604248046875,
+      "reward_std": 0.009487086907029152,
+      "rewards//mean": 0.75604248046875,
+      "rewards//std": 0.03158097714185715,
+      "step": 1311
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2624,
+      "grad_norm": 7.72947359085083,
+      "kl": 3.786453163251281,
+      "learning_rate": 8.482411448674445e-07,
+      "loss": 0.1515,
+      "num_tokens": 11340712.0,
+      "reward": 0.74591064453125,
+      "reward_std": 0.006868995260447264,
+      "rewards//mean": 0.74591064453125,
+      "rewards//std": 0.032109566032886505,
+      "step": 1312
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2626,
+      "grad_norm": 6.769286632537842,
+      "kl": 5.255178485065699,
+      "learning_rate": 8.480133651357505e-07,
+      "loss": 0.2102,
+      "num_tokens": 11349392.0,
+      "reward": 0.74322509765625,
+      "reward_std": 0.009714778512716293,
+      "rewards//mean": 0.74322509765625,
+      "rewards//std": 0.030680909752845764,
+      "step": 1313
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2628,
+      "grad_norm": 6.920955657958984,
+      "kl": 2.671338966116309,
+      "learning_rate": 8.477854452243147e-07,
+      "loss": 0.1069,
+      "num_tokens": 11358080.0,
+      "reward": 0.7919921875,
+      "reward_std": 0.01285955123603344,
+      "rewards//mean": 0.7919921875,
+      "rewards//std": 0.02309396117925644,
+      "step": 1314
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.263,
+      "grad_norm": 5.518157005310059,
+      "kl": 3.1050837635993958,
+      "learning_rate": 8.475573852249434e-07,
+      "loss": 0.1242,
+      "num_tokens": 11366696.0,
+      "reward": 0.74749755859375,
+      "reward_std": 0.008475309237837791,
+      "rewards//mean": 0.74749755859375,
+      "rewards//std": 0.02823164500296116,
+      "step": 1315
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2632,
+      "grad_norm": 9.080916404724121,
+      "kl": 2.1023597456514835,
+      "learning_rate": 8.473291852294986e-07,
+      "loss": 0.0841,
+      "num_tokens": 11375288.0,
+      "reward": 0.757080078125,
+      "reward_std": 0.006348649971187115,
+      "rewards//mean": 0.757080078125,
+      "rewards//std": 0.019548293203115463,
+      "step": 1316
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2634,
+      "grad_norm": 5.56795072555542,
+      "kl": 2.205072931945324,
+      "learning_rate": 8.471008453298996e-07,
+      "loss": 0.0882,
+      "num_tokens": 11383904.0,
+      "reward": 0.77349853515625,
+      "reward_std": 0.007874330505728722,
+      "rewards//mean": 0.77349853515625,
+      "rewards//std": 0.02834402211010456,
+      "step": 1317
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2636,
+      "grad_norm": 6.041005611419678,
+      "kl": 1.6031034365296364,
+      "learning_rate": 8.468723656181218e-07,
+      "loss": 0.0641,
+      "num_tokens": 11392448.0,
+      "reward": 0.7506103515625,
+      "reward_std": 0.005055404268205166,
+      "rewards//mean": 0.7506103515625,
+      "rewards//std": 0.022991150617599487,
+      "step": 1318
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2638,
+      "grad_norm": 3.836426258087158,
+      "kl": 1.4163221195340157,
+      "learning_rate": 8.466437461861964e-07,
+      "loss": 0.0567,
+      "num_tokens": 11401088.0,
+      "reward": 0.77691650390625,
+      "reward_std": 0.0033918290864676237,
+      "rewards//mean": 0.77691650390625,
+      "rewards//std": 0.019289391115307808,
+      "step": 1319
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.264,
+      "grad_norm": 5.417503356933594,
+      "kl": 4.588376244530082,
+      "learning_rate": 8.464149871262116e-07,
+      "loss": 0.1835,
+      "num_tokens": 11409840.0,
+      "reward": 0.75164794921875,
+      "reward_std": 0.00984761118888855,
+      "rewards//mean": 0.75164794921875,
+      "rewards//std": 0.04103292152285576,
+      "step": 1320
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2642,
+      "grad_norm": 2.41438889503479,
+      "kl": 2.309561477974057,
+      "learning_rate": 8.461860885303113e-07,
+      "loss": 0.0924,
+      "num_tokens": 11418512.0,
+      "reward": 0.74456787109375,
+      "reward_std": 0.009001797065138817,
+      "rewards//mean": 0.74456787109375,
+      "rewards//std": 0.029802938923239708,
+      "step": 1321
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2644,
+      "grad_norm": 3.3165946006774902,
+      "kl": 4.257082613185048,
+      "learning_rate": 8.459570504906961e-07,
+      "loss": 0.1703,
+      "num_tokens": 11427168.0,
+      "reward": 0.7557373046875,
+      "reward_std": 0.011855144053697586,
+      "rewards//mean": 0.7557373046875,
+      "rewards//std": 0.02796097658574581,
+      "step": 1322
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2646,
+      "grad_norm": 6.713140487670898,
+      "kl": 1.3529539816081524,
+      "learning_rate": 8.457278730996222e-07,
+      "loss": 0.0541,
+      "num_tokens": 11435776.0,
+      "reward": 0.8125,
+      "reward_std": 0.006250374019145966,
+      "rewards//mean": 0.8125,
+      "rewards//std": 0.01666027121245861,
+      "step": 1323
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2648,
+      "grad_norm": 8.603462219238281,
+      "kl": 4.811173427850008,
+      "learning_rate": 8.454985564494024e-07,
+      "loss": 0.1924,
+      "num_tokens": 11444424.0,
+      "reward": 0.7425537109375,
+      "reward_std": 0.0073181819170713425,
+      "rewards//mean": 0.7425537109375,
+      "rewards//std": 0.034532178193330765,
+      "step": 1324
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.265,
+      "grad_norm": 8.883708000183105,
+      "kl": 1.1453363094478846,
+      "learning_rate": 8.452691006324054e-07,
+      "loss": 0.0458,
+      "num_tokens": 11453048.0,
+      "reward": 0.74786376953125,
+      "reward_std": 0.0022728105541318655,
+      "rewards//mean": 0.74786376953125,
+      "rewards//std": 0.028670594096183777,
+      "step": 1325
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2652,
+      "grad_norm": 6.938794136047363,
+      "kl": 4.304388772696257,
+      "learning_rate": 8.45039505741056e-07,
+      "loss": 0.1722,
+      "num_tokens": 11461712.0,
+      "reward": 0.74566650390625,
+      "reward_std": 0.008371752686798573,
+      "rewards//mean": 0.74566650390625,
+      "rewards//std": 0.030931515619158745,
+      "step": 1326
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2654,
+      "grad_norm": 5.500860691070557,
+      "kl": 2.4787068367004395,
+      "learning_rate": 8.448097718678348e-07,
+      "loss": 0.0991,
+      "num_tokens": 11470416.0,
+      "reward": 0.7520751953125,
+      "reward_std": 0.006374950520694256,
+      "rewards//mean": 0.7520751953125,
+      "rewards//std": 0.03016122244298458,
+      "step": 1327
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2656,
+      "grad_norm": 4.477947235107422,
+      "kl": 2.00894608348608,
+      "learning_rate": 8.44579899105279e-07,
+      "loss": 0.0804,
+      "num_tokens": 11479032.0,
+      "reward": 0.75067138671875,
+      "reward_std": 0.0037859920412302017,
+      "rewards//mean": 0.75067138671875,
+      "rewards//std": 0.021379046142101288,
+      "step": 1328
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2658,
+      "grad_norm": 5.4936957359313965,
+      "kl": 1.6012097373604774,
+      "learning_rate": 8.443498875459808e-07,
+      "loss": 0.064,
+      "num_tokens": 11487568.0,
+      "reward": 0.72161865234375,
+      "reward_std": 0.0034891131799668074,
+      "rewards//mean": 0.72161865234375,
+      "rewards//std": 0.0310653205960989,
+      "step": 1329
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.266,
+      "grad_norm": 3.366276264190674,
+      "kl": 1.7269084453582764,
+      "learning_rate": 8.441197372825892e-07,
+      "loss": 0.0691,
+      "num_tokens": 11496160.0,
+      "reward": 0.73370361328125,
+      "reward_std": 0.004661104176193476,
+      "rewards//mean": 0.73370361328125,
+      "rewards//std": 0.032841384410858154,
+      "step": 1330
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2662,
+      "grad_norm": 4.2201056480407715,
+      "kl": 4.002227613702416,
+      "learning_rate": 8.438894484078085e-07,
+      "loss": 0.1601,
+      "num_tokens": 11504792.0,
+      "reward": 0.75140380859375,
+      "reward_std": 0.015436064451932907,
+      "rewards//mean": 0.75140380859375,
+      "rewards//std": 0.034746479243040085,
+      "step": 1331
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2664,
+      "grad_norm": 8.48907470703125,
+      "kl": 5.469716316089034,
+      "learning_rate": 8.43659021014399e-07,
+      "loss": 0.2188,
+      "num_tokens": 11513432.0,
+      "reward": 0.76434326171875,
+      "reward_std": 0.011993806809186935,
+      "rewards//mean": 0.76434326171875,
+      "rewards//std": 0.0229155533015728,
+      "step": 1332
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2666,
+      "grad_norm": 6.576357364654541,
+      "kl": 2.191881190985441,
+      "learning_rate": 8.434284551951772e-07,
+      "loss": 0.0877,
+      "num_tokens": 11522112.0,
+      "reward": 0.77593994140625,
+      "reward_std": 0.009596865624189377,
+      "rewards//mean": 0.77593994140625,
+      "rewards//std": 0.02598922699689865,
+      "step": 1333
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2668,
+      "grad_norm": 20.691680908203125,
+      "kl": 2.7848943509161472,
+      "learning_rate": 8.431977510430145e-07,
+      "loss": 0.1114,
+      "num_tokens": 11530648.0,
+      "reward": 0.72088623046875,
+      "reward_std": 0.00493771955370903,
+      "rewards//mean": 0.72088623046875,
+      "rewards//std": 0.03379638120532036,
+      "step": 1334
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.267,
+      "grad_norm": 8.741593360900879,
+      "kl": 3.956906611099839,
+      "learning_rate": 8.429669086508389e-07,
+      "loss": 0.1583,
+      "num_tokens": 11539376.0,
+      "reward": 0.7659912109375,
+      "reward_std": 0.009909494780004025,
+      "rewards//mean": 0.7659912109375,
+      "rewards//std": 0.040937863290309906,
+      "step": 1335
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2672,
+      "grad_norm": 19.778757095336914,
+      "kl": 4.849205795675516,
+      "learning_rate": 8.427359281116333e-07,
+      "loss": 0.194,
+      "num_tokens": 11548024.0,
+      "reward": 0.75140380859375,
+      "reward_std": 0.01166295912116766,
+      "rewards//mean": 0.75140380859375,
+      "rewards//std": 0.025117216631770134,
+      "step": 1336
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2674,
+      "grad_norm": 7.019726753234863,
+      "kl": 5.418259494006634,
+      "learning_rate": 8.42504809518437e-07,
+      "loss": 0.2167,
+      "num_tokens": 11556656.0,
+      "reward": 0.75677490234375,
+      "reward_std": 0.011645495891571045,
+      "rewards//mean": 0.75677490234375,
+      "rewards//std": 0.03187344968318939,
+      "step": 1337
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2676,
+      "grad_norm": 40.18573760986328,
+      "kl": 5.802844006568193,
+      "learning_rate": 8.422735529643443e-07,
+      "loss": 0.2321,
+      "num_tokens": 11565296.0,
+      "reward": 0.76519775390625,
+      "reward_std": 0.01373140700161457,
+      "rewards//mean": 0.76519775390625,
+      "rewards//std": 0.028306612744927406,
+      "step": 1338
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2678,
+      "grad_norm": 7.769367218017578,
+      "kl": 3.249770663678646,
+      "learning_rate": 8.420421585425055e-07,
+      "loss": 0.13,
+      "num_tokens": 11573864.0,
+      "reward": 0.74432373046875,
+      "reward_std": 0.010364457033574581,
+      "rewards//mean": 0.74432373046875,
+      "rewards//std": 0.033457860350608826,
+      "step": 1339
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.268,
+      "grad_norm": 20.78012466430664,
+      "kl": 7.734453845769167,
+      "learning_rate": 8.41810626346126e-07,
+      "loss": 0.3094,
+      "num_tokens": 11582424.0,
+      "reward": 0.73883056640625,
+      "reward_std": 0.014386819675564766,
+      "rewards//mean": 0.73883056640625,
+      "rewards//std": 0.03075679950416088,
+      "step": 1340
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2682,
+      "grad_norm": 43.48139190673828,
+      "kl": 6.283532673493028,
+      "learning_rate": 8.415789564684673e-07,
+      "loss": 0.2513,
+      "num_tokens": 11590968.0,
+      "reward": 0.7933349609375,
+      "reward_std": 0.010344470851123333,
+      "rewards//mean": 0.7933349609375,
+      "rewards//std": 0.025178389623761177,
+      "step": 1341
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2684,
+      "grad_norm": 60.180294036865234,
+      "kl": 10.318531692028046,
+      "learning_rate": 8.413471490028455e-07,
+      "loss": 0.4127,
+      "num_tokens": 11599640.0,
+      "reward": 0.76593017578125,
+      "reward_std": 0.015049831941723824,
+      "rewards//mean": 0.76593017578125,
+      "rewards//std": 0.02918447181582451,
+      "step": 1342
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2686,
+      "grad_norm": 60.8212890625,
+      "kl": 9.179529754444957,
+      "learning_rate": 8.41115204042633e-07,
+      "loss": 0.3672,
+      "num_tokens": 11608232.0,
+      "reward": 0.749267578125,
+      "reward_std": 0.013325735926628113,
+      "rewards//mean": 0.749267578125,
+      "rewards//std": 0.03470119461417198,
+      "step": 1343
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2688,
+      "grad_norm": 38.94044494628906,
+      "kl": 5.907689101994038,
+      "learning_rate": 8.408831216812573e-07,
+      "loss": 0.2363,
+      "num_tokens": 11616920.0,
+      "reward": 0.758544921875,
+      "reward_std": 0.015807736665010452,
+      "rewards//mean": 0.758544921875,
+      "rewards//std": 0.04591904953122139,
+      "step": 1344
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.269,
+      "grad_norm": 18.94415855407715,
+      "kl": 3.504896441474557,
+      "learning_rate": 8.406509020122008e-07,
+      "loss": 0.1402,
+      "num_tokens": 11625560.0,
+      "reward": 0.77899169921875,
+      "reward_std": 0.0050090402364730835,
+      "rewards//mean": 0.77899169921875,
+      "rewards//std": 0.026509970426559448,
+      "step": 1345
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2692,
+      "grad_norm": 26.979238510131836,
+      "kl": 6.458527412265539,
+      "learning_rate": 8.404185451290017e-07,
+      "loss": 0.2583,
+      "num_tokens": 11634256.0,
+      "reward": 0.7689208984375,
+      "reward_std": 0.012575831264257431,
+      "rewards//mean": 0.7689208984375,
+      "rewards//std": 0.03487763553857803,
+      "step": 1346
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2694,
+      "grad_norm": 5.83043909072876,
+      "kl": 4.26373004168272,
+      "learning_rate": 8.401860511252533e-07,
+      "loss": 0.1705,
+      "num_tokens": 11642840.0,
+      "reward": 0.75396728515625,
+      "reward_std": 0.00985700823366642,
+      "rewards//mean": 0.75396728515625,
+      "rewards//std": 0.03178403899073601,
+      "step": 1347
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2696,
+      "grad_norm": 5.888754367828369,
+      "kl": 2.50071469694376,
+      "learning_rate": 8.399534200946043e-07,
+      "loss": 0.1,
+      "num_tokens": 11651408.0,
+      "reward": 0.74603271484375,
+      "reward_std": 0.006771939806640148,
+      "rewards//mean": 0.74603271484375,
+      "rewards//std": 0.02661653608083725,
+      "step": 1348
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2698,
+      "grad_norm": 11.314641952514648,
+      "kl": 2.699692741036415,
+      "learning_rate": 8.397206521307583e-07,
+      "loss": 0.108,
+      "num_tokens": 11660048.0,
+      "reward": 0.7738037109375,
+      "reward_std": 0.010594477877020836,
+      "rewards//mean": 0.7738037109375,
+      "rewards//std": 0.025293558835983276,
+      "step": 1349
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.27,
+      "grad_norm": 8.233244895935059,
+      "kl": 2.063253328204155,
+      "learning_rate": 8.394877473274741e-07,
+      "loss": 0.0825,
+      "num_tokens": 11668728.0,
+      "reward": 0.74749755859375,
+      "reward_std": 0.0059649404138326645,
+      "rewards//mean": 0.74749755859375,
+      "rewards//std": 0.03423384577035904,
+      "step": 1350
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2702,
+      "grad_norm": 4.355931758880615,
+      "kl": 1.0865337681025267,
+      "learning_rate": 8.392547057785661e-07,
+      "loss": 0.0435,
+      "num_tokens": 11677376.0,
+      "reward": 0.77728271484375,
+      "reward_std": 0.0041401139460504055,
+      "rewards//mean": 0.77728271484375,
+      "rewards//std": 0.018616894260048866,
+      "step": 1351
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2704,
+      "grad_norm": 5.585280895233154,
+      "kl": 2.5505129732191563,
+      "learning_rate": 8.39021527577903e-07,
+      "loss": 0.102,
+      "num_tokens": 11686016.0,
+      "reward": 0.776611328125,
+      "reward_std": 0.009554125368595123,
+      "rewards//mean": 0.776611328125,
+      "rewards//std": 0.022259533405303955,
+      "step": 1352
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2706,
+      "grad_norm": 23.33195686340332,
+      "kl": 3.4384165573865175,
+      "learning_rate": 8.387882128194092e-07,
+      "loss": 0.1375,
+      "num_tokens": 11694712.0,
+      "reward": 0.78271484375,
+      "reward_std": 0.008728314191102982,
+      "rewards//mean": 0.78271484375,
+      "rewards//std": 0.03264866769313812,
+      "step": 1353
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2708,
+      "grad_norm": 16.409711837768555,
+      "kl": 1.1684235334396362,
+      "learning_rate": 8.385547615970638e-07,
+      "loss": 0.0467,
+      "num_tokens": 11703384.0,
+      "reward": 0.73870849609375,
+      "reward_std": 0.0020206072367727757,
+      "rewards//mean": 0.73870849609375,
+      "rewards//std": 0.018676968291401863,
+      "step": 1354
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.271,
+      "grad_norm": 6.225190162658691,
+      "kl": 2.5341316740959883,
+      "learning_rate": 8.38321174004901e-07,
+      "loss": 0.1014,
+      "num_tokens": 11712016.0,
+      "reward": 0.77471923828125,
+      "reward_std": 0.0039366185665130615,
+      "rewards//mean": 0.77471923828125,
+      "rewards//std": 0.020857207477092743,
+      "step": 1355
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2712,
+      "grad_norm": 11.49312973022461,
+      "kl": 3.6324777752161026,
+      "learning_rate": 8.380874501370097e-07,
+      "loss": 0.1453,
+      "num_tokens": 11720648.0,
+      "reward": 0.7725830078125,
+      "reward_std": 0.014991494826972485,
+      "rewards//mean": 0.7725830078125,
+      "rewards//std": 0.03184476122260094,
+      "step": 1356
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2714,
+      "grad_norm": 1.3001525402069092,
+      "kl": 1.5222731325775385,
+      "learning_rate": 8.378535900875338e-07,
+      "loss": 0.0609,
+      "num_tokens": 11729304.0,
+      "reward": 0.73431396484375,
+      "reward_std": 0.002391017973423004,
+      "rewards//mean": 0.73431396484375,
+      "rewards//std": 0.02544415183365345,
+      "step": 1357
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2716,
+      "grad_norm": 8.484679222106934,
+      "kl": 2.7289376724511385,
+      "learning_rate": 8.376195939506725e-07,
+      "loss": 0.1092,
+      "num_tokens": 11738040.0,
+      "reward": 0.74029541015625,
+      "reward_std": 0.00747756939381361,
+      "rewards//mean": 0.74029541015625,
+      "rewards//std": 0.03327684476971626,
+      "step": 1358
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2718,
+      "grad_norm": 18.6961727142334,
+      "kl": 4.8908359333872795,
+      "learning_rate": 8.373854618206789e-07,
+      "loss": 0.1956,
+      "num_tokens": 11746752.0,
+      "reward": 0.73046875,
+      "reward_std": 0.02031049132347107,
+      "rewards//mean": 0.73046875,
+      "rewards//std": 0.04448913410305977,
+      "step": 1359
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.272,
+      "grad_norm": 19.100324630737305,
+      "kl": 2.1134145110845566,
+      "learning_rate": 8.371511937918617e-07,
+      "loss": 0.0845,
+      "num_tokens": 11755448.0,
+      "reward": 0.72198486328125,
+      "reward_std": 0.007530339062213898,
+      "rewards//mean": 0.72198486328125,
+      "rewards//std": 0.030421772971749306,
+      "step": 1360
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2722,
+      "grad_norm": 11.395090103149414,
+      "kl": 2.889318846166134,
+      "learning_rate": 8.369167899585839e-07,
+      "loss": 0.1156,
+      "num_tokens": 11764152.0,
+      "reward": 0.75225830078125,
+      "reward_std": 0.008851965889334679,
+      "rewards//mean": 0.75225830078125,
+      "rewards//std": 0.029502255842089653,
+      "step": 1361
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2724,
+      "grad_norm": 14.356400489807129,
+      "kl": 1.5334529355168343,
+      "learning_rate": 8.366822504152636e-07,
+      "loss": 0.0613,
+      "num_tokens": 11772776.0,
+      "reward": 0.73876953125,
+      "reward_std": 0.00911199301481247,
+      "rewards//mean": 0.73876953125,
+      "rewards//std": 0.026157667860388756,
+      "step": 1362
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2726,
+      "grad_norm": 8.904847145080566,
+      "kl": 2.355218816548586,
+      "learning_rate": 8.364475752563728e-07,
+      "loss": 0.0942,
+      "num_tokens": 11781408.0,
+      "reward": 0.75860595703125,
+      "reward_std": 0.009294064715504646,
+      "rewards//mean": 0.75860595703125,
+      "rewards//std": 0.034303195774555206,
+      "step": 1363
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2728,
+      "grad_norm": 18.4609317779541,
+      "kl": 1.0844163838773966,
+      "learning_rate": 8.362127645764389e-07,
+      "loss": 0.0434,
+      "num_tokens": 11789976.0,
+      "reward": 0.74090576171875,
+      "reward_std": 0.006823434494435787,
+      "rewards//mean": 0.74090576171875,
+      "rewards//std": 0.027805369347333908,
+      "step": 1364
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.273,
+      "grad_norm": 35.80815887451172,
+      "kl": 8.231054339557886,
+      "learning_rate": 8.359778184700439e-07,
+      "loss": 0.3292,
+      "num_tokens": 11798632.0,
+      "reward": 0.78948974609375,
+      "reward_std": 0.014547184109687805,
+      "rewards//mean": 0.78948974609375,
+      "rewards//std": 0.02923266962170601,
+      "step": 1365
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2732,
+      "grad_norm": 46.54033660888672,
+      "kl": 5.669671170413494,
+      "learning_rate": 8.357427370318238e-07,
+      "loss": 0.2268,
+      "num_tokens": 11807256.0,
+      "reward": 0.7642822265625,
+      "reward_std": 0.01499166153371334,
+      "rewards//mean": 0.7642822265625,
+      "rewards//std": 0.028656136244535446,
+      "step": 1366
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2734,
+      "grad_norm": 49.29010009765625,
+      "kl": 7.49124613031745,
+      "learning_rate": 8.355075203564692e-07,
+      "loss": 0.2996,
+      "num_tokens": 11815920.0,
+      "reward": 0.74505615234375,
+      "reward_std": 0.010848349891602993,
+      "rewards//mean": 0.74505615234375,
+      "rewards//std": 0.0375593900680542,
+      "step": 1367
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2736,
+      "grad_norm": 31.461627960205078,
+      "kl": 4.508681073784828,
+      "learning_rate": 8.352721685387256e-07,
+      "loss": 0.1803,
+      "num_tokens": 11824560.0,
+      "reward": 0.75567626953125,
+      "reward_std": 0.008087869733572006,
+      "rewards//mean": 0.75567626953125,
+      "rewards//std": 0.027734503149986267,
+      "step": 1368
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2738,
+      "grad_norm": 15.764370918273926,
+      "kl": 5.868150206282735,
+      "learning_rate": 8.350366816733926e-07,
+      "loss": 0.2347,
+      "num_tokens": 11833184.0,
+      "reward": 0.74560546875,
+      "reward_std": 0.00955010112375021,
+      "rewards//mean": 0.74560546875,
+      "rewards//std": 0.02954077534377575,
+      "step": 1369
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.274,
+      "grad_norm": 21.798931121826172,
+      "kl": 1.0845754779875278,
+      "learning_rate": 8.348010598553243e-07,
+      "loss": 0.0434,
+      "num_tokens": 11841856.0,
+      "reward": 0.7723388671875,
+      "reward_std": 0.005296029150485992,
+      "rewards//mean": 0.7723388671875,
+      "rewards//std": 0.022845840081572533,
+      "step": 1370
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2742,
+      "grad_norm": 10.106986999511719,
+      "kl": 3.576843526214361,
+      "learning_rate": 8.34565303179429e-07,
+      "loss": 0.1431,
+      "num_tokens": 11850632.0,
+      "reward": 0.7630615234375,
+      "reward_std": 0.011000830680131912,
+      "rewards//mean": 0.7630615234375,
+      "rewards//std": 0.029158851131796837,
+      "step": 1371
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2744,
+      "grad_norm": 9.401496887207031,
+      "kl": 2.606949105858803,
+      "learning_rate": 8.343294117406698e-07,
+      "loss": 0.1043,
+      "num_tokens": 11859328.0,
+      "reward": 0.7569580078125,
+      "reward_std": 0.0065363505855202675,
+      "rewards//mean": 0.7569580078125,
+      "rewards//std": 0.0202970989048481,
+      "step": 1372
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2746,
+      "grad_norm": 9.911111831665039,
+      "kl": 2.0001470036804676,
+      "learning_rate": 8.340933856340635e-07,
+      "loss": 0.08,
+      "num_tokens": 11867920.0,
+      "reward": 0.73931884765625,
+      "reward_std": 0.008149401284754276,
+      "rewards//mean": 0.73931884765625,
+      "rewards//std": 0.03379145264625549,
+      "step": 1373
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2748,
+      "grad_norm": 13.342793464660645,
+      "kl": 3.7259758468717337,
+      "learning_rate": 8.338572249546812e-07,
+      "loss": 0.149,
+      "num_tokens": 11876616.0,
+      "reward": 0.75372314453125,
+      "reward_std": 0.00650325370952487,
+      "rewards//mean": 0.75372314453125,
+      "rewards//std": 0.0354849174618721,
+      "step": 1374
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.275,
+      "grad_norm": 8.230053901672363,
+      "kl": 2.122542344033718,
+      "learning_rate": 8.336209297976489e-07,
+      "loss": 0.0849,
+      "num_tokens": 11885248.0,
+      "reward": 0.77911376953125,
+      "reward_std": 0.010866462253034115,
+      "rewards//mean": 0.77911376953125,
+      "rewards//std": 0.018570488318800926,
+      "step": 1375
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2752,
+      "grad_norm": 7.580397129058838,
+      "kl": 5.059437431395054,
+      "learning_rate": 8.333845002581458e-07,
+      "loss": 0.2024,
+      "num_tokens": 11893872.0,
+      "reward": 0.76678466796875,
+      "reward_std": 0.017937738448381424,
+      "rewards//mean": 0.76678466796875,
+      "rewards//std": 0.03117768093943596,
+      "step": 1376
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2754,
+      "grad_norm": 6.89148473739624,
+      "kl": 4.216024033725262,
+      "learning_rate": 8.331479364314059e-07,
+      "loss": 0.1686,
+      "num_tokens": 11902448.0,
+      "reward": 0.74981689453125,
+      "reward_std": 0.015681136399507523,
+      "rewards//mean": 0.74981689453125,
+      "rewards//std": 0.03388584405183792,
+      "step": 1377
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2756,
+      "grad_norm": 3.76511549949646,
+      "kl": 3.6495047602802515,
+      "learning_rate": 8.32911238412717e-07,
+      "loss": 0.146,
+      "num_tokens": 11911064.0,
+      "reward": 0.7918701171875,
+      "reward_std": 0.010602817870676517,
+      "rewards//mean": 0.7918701171875,
+      "rewards//std": 0.02308838814496994,
+      "step": 1378
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2758,
+      "grad_norm": 8.92732048034668,
+      "kl": 2.4589269440621138,
+      "learning_rate": 8.326744062974211e-07,
+      "loss": 0.0984,
+      "num_tokens": 11919760.0,
+      "reward": 0.7510986328125,
+      "reward_std": 0.011387133970856667,
+      "rewards//mean": 0.7510986328125,
+      "rewards//std": 0.02497313730418682,
+      "step": 1379
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.276,
+      "grad_norm": 8.535314559936523,
+      "kl": 1.194465760141611,
+      "learning_rate": 8.324374401809142e-07,
+      "loss": 0.0478,
+      "num_tokens": 11928344.0,
+      "reward": 0.73162841796875,
+      "reward_std": 0.0032463876996189356,
+      "rewards//mean": 0.73162841796875,
+      "rewards//std": 0.03741361200809479,
+      "step": 1380
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2762,
+      "grad_norm": 3.3985280990600586,
+      "kl": 3.09967577457428,
+      "learning_rate": 8.322003401586461e-07,
+      "loss": 0.124,
+      "num_tokens": 11936976.0,
+      "reward": 0.786865234375,
+      "reward_std": 0.013017643243074417,
+      "rewards//mean": 0.786865234375,
+      "rewards//std": 0.03151145577430725,
+      "step": 1381
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2764,
+      "grad_norm": 6.333940029144287,
+      "kl": 2.7226822581142187,
+      "learning_rate": 8.319631063261207e-07,
+      "loss": 0.1089,
+      "num_tokens": 11945576.0,
+      "reward": 0.75115966796875,
+      "reward_std": 0.006218246184289455,
+      "rewards//mean": 0.75115966796875,
+      "rewards//std": 0.021773340180516243,
+      "step": 1382
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2766,
+      "grad_norm": 0.2580198347568512,
+      "kl": 0.5196405425667763,
+      "learning_rate": 8.317257387788958e-07,
+      "loss": 0.0208,
+      "num_tokens": 11954136.0,
+      "reward": 0.74786376953125,
+      "reward_std": 0.0005179004510864615,
+      "rewards//mean": 0.74786376953125,
+      "rewards//std": 0.021484991535544395,
+      "step": 1383
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2768,
+      "grad_norm": 4.238036155700684,
+      "kl": 4.758801337331533,
+      "learning_rate": 8.314882376125831e-07,
+      "loss": 0.1904,
+      "num_tokens": 11962824.0,
+      "reward": 0.7857666015625,
+      "reward_std": 0.01685452274978161,
+      "rewards//mean": 0.7857666015625,
+      "rewards//std": 0.03431759029626846,
+      "step": 1384
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.277,
+      "grad_norm": 1.8679053783416748,
+      "kl": 1.3025844171643257,
+      "learning_rate": 8.312506029228477e-07,
+      "loss": 0.0521,
+      "num_tokens": 11971504.0,
+      "reward": 0.7469482421875,
+      "reward_std": 0.003025681944563985,
+      "rewards//mean": 0.7469482421875,
+      "rewards//std": 0.026977859437465668,
+      "step": 1385
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2772,
+      "grad_norm": 4.578019618988037,
+      "kl": 3.4632092732936144,
+      "learning_rate": 8.310128348054093e-07,
+      "loss": 0.1385,
+      "num_tokens": 11980152.0,
+      "reward": 0.758056640625,
+      "reward_std": 0.006674261763691902,
+      "rewards//mean": 0.758056640625,
+      "rewards//std": 0.03284374624490738,
+      "step": 1386
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2774,
+      "grad_norm": 2.8782236576080322,
+      "kl": 1.7442466355860233,
+      "learning_rate": 8.307749333560404e-07,
+      "loss": 0.0698,
+      "num_tokens": 11988744.0,
+      "reward": 0.77886962890625,
+      "reward_std": 0.005287739913910627,
+      "rewards//mean": 0.77886962890625,
+      "rewards//std": 0.018487975001335144,
+      "step": 1387
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2776,
+      "grad_norm": 3.385528087615967,
+      "kl": 1.5798235796391964,
+      "learning_rate": 8.305368986705681e-07,
+      "loss": 0.0632,
+      "num_tokens": 11997312.0,
+      "reward": 0.73919677734375,
+      "reward_std": 0.004757335409522057,
+      "rewards//mean": 0.73919677734375,
+      "rewards//std": 0.033995117992162704,
+      "step": 1388
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2778,
+      "grad_norm": 6.672630310058594,
+      "kl": 2.228483470156789,
+      "learning_rate": 8.302987308448723e-07,
+      "loss": 0.0891,
+      "num_tokens": 12005928.0,
+      "reward": 0.75494384765625,
+      "reward_std": 0.003914062865078449,
+      "rewards//mean": 0.75494384765625,
+      "rewards//std": 0.034542862325906754,
+      "step": 1389
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.278,
+      "grad_norm": 1.9289408922195435,
+      "kl": 0.9908090904355049,
+      "learning_rate": 8.300604299748874e-07,
+      "loss": 0.0396,
+      "num_tokens": 12014688.0,
+      "reward": 0.7354736328125,
+      "reward_std": 0.001987332245334983,
+      "rewards//mean": 0.7354736328125,
+      "rewards//std": 0.02719247341156006,
+      "step": 1390
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2782,
+      "grad_norm": 20.58250617980957,
+      "kl": 3.8105711452662945,
+      "learning_rate": 8.298219961566008e-07,
+      "loss": 0.1524,
+      "num_tokens": 12023352.0,
+      "reward": 0.74493408203125,
+      "reward_std": 0.014214625582098961,
+      "rewards//mean": 0.74493408203125,
+      "rewards//std": 0.03664836660027504,
+      "step": 1391
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2784,
+      "grad_norm": 7.755046367645264,
+      "kl": 5.07947096042335,
+      "learning_rate": 8.295834294860534e-07,
+      "loss": 0.2032,
+      "num_tokens": 12031984.0,
+      "reward": 0.75604248046875,
+      "reward_std": 0.009290643967688084,
+      "rewards//mean": 0.75604248046875,
+      "rewards//std": 0.02914814092218876,
+      "step": 1392
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2786,
+      "grad_norm": 3.1130611896514893,
+      "kl": 3.0101313907653093,
+      "learning_rate": 8.293447300593402e-07,
+      "loss": 0.1204,
+      "num_tokens": 12040632.0,
+      "reward": 0.7606201171875,
+      "reward_std": 0.008227781392633915,
+      "rewards//mean": 0.7606201171875,
+      "rewards//std": 0.02051076479256153,
+      "step": 1393
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2788,
+      "grad_norm": 5.169640064239502,
+      "kl": 5.0863844864070415,
+      "learning_rate": 8.291058979726091e-07,
+      "loss": 0.2035,
+      "num_tokens": 12049192.0,
+      "reward": 0.7628173828125,
+      "reward_std": 0.008807415142655373,
+      "rewards//mean": 0.7628173828125,
+      "rewards//std": 0.030149174854159355,
+      "step": 1394
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.279,
+      "grad_norm": 3.1735270023345947,
+      "kl": 3.724377514794469,
+      "learning_rate": 8.288669333220614e-07,
+      "loss": 0.149,
+      "num_tokens": 12057776.0,
+      "reward": 0.7547607421875,
+      "reward_std": 0.009010215289890766,
+      "rewards//mean": 0.7547607421875,
+      "rewards//std": 0.015096287243068218,
+      "step": 1395
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2792,
+      "grad_norm": 7.62028694152832,
+      "kl": 6.1161501705646515,
+      "learning_rate": 8.286278362039527e-07,
+      "loss": 0.2446,
+      "num_tokens": 12066448.0,
+      "reward": 0.7623291015625,
+      "reward_std": 0.010623966343700886,
+      "rewards//mean": 0.7623291015625,
+      "rewards//std": 0.024766186252236366,
+      "step": 1396
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2794,
+      "grad_norm": 1.2890839576721191,
+      "kl": 1.417680436745286,
+      "learning_rate": 8.283886067145906e-07,
+      "loss": 0.0567,
+      "num_tokens": 12075056.0,
+      "reward": 0.7945556640625,
+      "reward_std": 0.005159334745258093,
+      "rewards//mean": 0.7945556640625,
+      "rewards//std": 0.024623975157737732,
+      "step": 1397
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2796,
+      "grad_norm": 3.1143851280212402,
+      "kl": 2.9287510327994823,
+      "learning_rate": 8.281492449503372e-07,
+      "loss": 0.1172,
+      "num_tokens": 12083688.0,
+      "reward": 0.7630615234375,
+      "reward_std": 0.005025902297347784,
+      "rewards//mean": 0.7630615234375,
+      "rewards//std": 0.023849500343203545,
+      "step": 1398
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2798,
+      "grad_norm": 5.231459140777588,
+      "kl": 3.7067789547145367,
+      "learning_rate": 8.279097510076069e-07,
+      "loss": 0.1483,
+      "num_tokens": 12092360.0,
+      "reward": 0.779296875,
+      "reward_std": 0.006181907840073109,
+      "rewards//mean": 0.779296875,
+      "rewards//std": 0.027116939425468445,
+      "step": 1399
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.28,
+      "grad_norm": 2.3263142108917236,
+      "kl": 3.072752768173814,
+      "learning_rate": 8.276701249828684e-07,
+      "loss": 0.1229,
+      "num_tokens": 12100960.0,
+      "reward": 0.75860595703125,
+      "reward_std": 0.009692623279988766,
+      "rewards//mean": 0.75860595703125,
+      "rewards//std": 0.03468237817287445,
+      "step": 1400
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2802,
+      "grad_norm": 22.592851638793945,
+      "kl": 1.1277550496160984,
+      "learning_rate": 8.274303669726426e-07,
+      "loss": 0.0451,
+      "num_tokens": 12109584.0,
+      "reward": 0.7689208984375,
+      "reward_std": 0.004550518933683634,
+      "rewards//mean": 0.7689208984375,
+      "rewards//std": 0.036059003323316574,
+      "step": 1401
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2804,
+      "grad_norm": 2.93373441696167,
+      "kl": 3.0542775746434927,
+      "learning_rate": 8.271904770735041e-07,
+      "loss": 0.1222,
+      "num_tokens": 12118208.0,
+      "reward": 0.77899169921875,
+      "reward_std": 0.0068513015285134315,
+      "rewards//mean": 0.77899169921875,
+      "rewards//std": 0.022965705022215843,
+      "step": 1402
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2806,
+      "grad_norm": 25.050630569458008,
+      "kl": 2.522257875651121,
+      "learning_rate": 8.269504553820805e-07,
+      "loss": 0.1009,
+      "num_tokens": 12126832.0,
+      "reward": 0.79827880859375,
+      "reward_std": 0.008362634107470512,
+      "rewards//mean": 0.79827880859375,
+      "rewards//std": 0.021703006699681282,
+      "step": 1403
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2808,
+      "grad_norm": 10.785796165466309,
+      "kl": 7.249267783015966,
+      "learning_rate": 8.267103019950528e-07,
+      "loss": 0.29,
+      "num_tokens": 12135432.0,
+      "reward": 0.72796630859375,
+      "reward_std": 0.01497596874833107,
+      "rewards//mean": 0.72796630859375,
+      "rewards//std": 0.03942888230085373,
+      "step": 1404
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.281,
+      "grad_norm": 6.517972469329834,
+      "kl": 4.577779654413462,
+      "learning_rate": 8.264700170091543e-07,
+      "loss": 0.1831,
+      "num_tokens": 12143992.0,
+      "reward": 0.75030517578125,
+      "reward_std": 0.00809216033667326,
+      "rewards//mean": 0.75030517578125,
+      "rewards//std": 0.019304296001791954,
+      "step": 1405
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2812,
+      "grad_norm": 31.89670753479004,
+      "kl": 9.678798243403435,
+      "learning_rate": 8.262296005211721e-07,
+      "loss": 0.3872,
+      "num_tokens": 12152592.0,
+      "reward": 0.74310302734375,
+      "reward_std": 0.005614377558231354,
+      "rewards//mean": 0.74310302734375,
+      "rewards//std": 0.027715394273400307,
+      "step": 1406
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2814,
+      "grad_norm": 32.459800720214844,
+      "kl": 7.412335563451052,
+      "learning_rate": 8.259890526279459e-07,
+      "loss": 0.2965,
+      "num_tokens": 12161296.0,
+      "reward": 0.77996826171875,
+      "reward_std": 0.011843318119645119,
+      "rewards//mean": 0.77996826171875,
+      "rewards//std": 0.023334499448537827,
+      "step": 1407
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2816,
+      "grad_norm": 9.2677001953125,
+      "kl": 6.327021928504109,
+      "learning_rate": 8.257483734263681e-07,
+      "loss": 0.2531,
+      "num_tokens": 12169976.0,
+      "reward": 0.72503662109375,
+      "reward_std": 0.015490911900997162,
+      "rewards//mean": 0.72503662109375,
+      "rewards//std": 0.04194832220673561,
+      "step": 1408
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2818,
+      "grad_norm": 8.746179580688477,
+      "kl": 1.0971181858330965,
+      "learning_rate": 8.255075630133845e-07,
+      "loss": 0.0439,
+      "num_tokens": 12178552.0,
+      "reward": 0.77410888671875,
+      "reward_std": 0.0032300525344908237,
+      "rewards//mean": 0.77410888671875,
+      "rewards//std": 0.018170008435845375,
+      "step": 1409
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.282,
+      "grad_norm": 19.859661102294922,
+      "kl": 3.708829317241907,
+      "learning_rate": 8.252666214859934e-07,
+      "loss": 0.1484,
+      "num_tokens": 12187224.0,
+      "reward": 0.7470703125,
+      "reward_std": 0.01625164970755577,
+      "rewards//mean": 0.7470703125,
+      "rewards//std": 0.03463919460773468,
+      "step": 1410
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2822,
+      "grad_norm": 9.35738468170166,
+      "kl": 4.432959834113717,
+      "learning_rate": 8.250255489412462e-07,
+      "loss": 0.1773,
+      "num_tokens": 12195976.0,
+      "reward": 0.76312255859375,
+      "reward_std": 0.009246166795492172,
+      "rewards//mean": 0.76312255859375,
+      "rewards//std": 0.0362609401345253,
+      "step": 1411
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2824,
+      "grad_norm": 14.963501930236816,
+      "kl": 5.9976518508046865,
+      "learning_rate": 8.247843454762466e-07,
+      "loss": 0.2399,
+      "num_tokens": 12204544.0,
+      "reward": 0.764404296875,
+      "reward_std": 0.018967021256685257,
+      "rewards//mean": 0.764404296875,
+      "rewards//std": 0.03559018298983574,
+      "step": 1412
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2826,
+      "grad_norm": 8.764498710632324,
+      "kl": 3.0336758885532618,
+      "learning_rate": 8.245430111881517e-07,
+      "loss": 0.1213,
+      "num_tokens": 12213224.0,
+      "reward": 0.78759765625,
+      "reward_std": 0.019061461091041565,
+      "rewards//mean": 0.78759765625,
+      "rewards//std": 0.032170381397008896,
+      "step": 1413
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2828,
+      "grad_norm": 10.85020923614502,
+      "kl": 4.418082553893328,
+      "learning_rate": 8.243015461741706e-07,
+      "loss": 0.1767,
+      "num_tokens": 12221920.0,
+      "reward": 0.75579833984375,
+      "reward_std": 0.008593752048909664,
+      "rewards//mean": 0.75579833984375,
+      "rewards//std": 0.03110233135521412,
+      "step": 1414
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.283,
+      "grad_norm": 2.4739699363708496,
+      "kl": 1.2441857382655144,
+      "learning_rate": 8.240599505315654e-07,
+      "loss": 0.0498,
+      "num_tokens": 12230600.0,
+      "reward": 0.77001953125,
+      "reward_std": 0.004998262505978346,
+      "rewards//mean": 0.77001953125,
+      "rewards//std": 0.033214978873729706,
+      "step": 1415
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2832,
+      "grad_norm": 6.969664096832275,
+      "kl": 4.846530532464385,
+      "learning_rate": 8.238182243576511e-07,
+      "loss": 0.1939,
+      "num_tokens": 12239248.0,
+      "reward": 0.767822265625,
+      "reward_std": 0.010738326236605644,
+      "rewards//mean": 0.767822265625,
+      "rewards//std": 0.024850696325302124,
+      "step": 1416
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2834,
+      "grad_norm": 18.58360481262207,
+      "kl": 3.6598623655736446,
+      "learning_rate": 8.235763677497945e-07,
+      "loss": 0.1464,
+      "num_tokens": 12247824.0,
+      "reward": 0.793701171875,
+      "reward_std": 0.008233538828790188,
+      "rewards//mean": 0.793701171875,
+      "rewards//std": 0.025179890915751457,
+      "step": 1417
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2836,
+      "grad_norm": 5.1898627281188965,
+      "kl": 3.482665505260229,
+      "learning_rate": 8.233343808054157e-07,
+      "loss": 0.1393,
+      "num_tokens": 12256496.0,
+      "reward": 0.750732421875,
+      "reward_std": 0.00788567028939724,
+      "rewards//mean": 0.750732421875,
+      "rewards//std": 0.03515194356441498,
+      "step": 1418
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2838,
+      "grad_norm": 5.801996231079102,
+      "kl": 1.4907474257051945,
+      "learning_rate": 8.23092263621987e-07,
+      "loss": 0.0596,
+      "num_tokens": 12265152.0,
+      "reward": 0.75091552734375,
+      "reward_std": 0.004526809323579073,
+      "rewards//mean": 0.75091552734375,
+      "rewards//std": 0.03150274604558945,
+      "step": 1419
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.284,
+      "grad_norm": 11.066574096679688,
+      "kl": 3.5229289308190346,
+      "learning_rate": 8.228500162970332e-07,
+      "loss": 0.1409,
+      "num_tokens": 12273816.0,
+      "reward": 0.7413330078125,
+      "reward_std": 0.011826589703559875,
+      "rewards//mean": 0.7413330078125,
+      "rewards//std": 0.040675222873687744,
+      "step": 1420
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2842,
+      "grad_norm": 14.3084716796875,
+      "kl": 4.823999624699354,
+      "learning_rate": 8.226076389281314e-07,
+      "loss": 0.193,
+      "num_tokens": 12282528.0,
+      "reward": 0.7667236328125,
+      "reward_std": 0.014889299869537354,
+      "rewards//mean": 0.7667236328125,
+      "rewards//std": 0.031392816454172134,
+      "step": 1421
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2844,
+      "grad_norm": 6.342036724090576,
+      "kl": 3.629468122497201,
+      "learning_rate": 8.223651316129114e-07,
+      "loss": 0.1452,
+      "num_tokens": 12291168.0,
+      "reward": 0.72991943359375,
+      "reward_std": 0.014151573181152344,
+      "rewards//mean": 0.72991943359375,
+      "rewards//std": 0.024762749671936035,
+      "step": 1422
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2846,
+      "grad_norm": 3.0061469078063965,
+      "kl": 2.580713367089629,
+      "learning_rate": 8.221224944490548e-07,
+      "loss": 0.1032,
+      "num_tokens": 12299824.0,
+      "reward": 0.76361083984375,
+      "reward_std": 0.006728575564920902,
+      "rewards//mean": 0.76361083984375,
+      "rewards//std": 0.03074203059077263,
+      "step": 1423
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2848,
+      "grad_norm": 6.261436462402344,
+      "kl": 2.5072155445814133,
+      "learning_rate": 8.21879727534296e-07,
+      "loss": 0.1003,
+      "num_tokens": 12308464.0,
+      "reward": 0.77105712890625,
+      "reward_std": 0.004622997250407934,
+      "rewards//mean": 0.77105712890625,
+      "rewards//std": 0.017071232199668884,
+      "step": 1424
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.285,
+      "grad_norm": 6.13707160949707,
+      "kl": 1.569300588220358,
+      "learning_rate": 8.216368309664213e-07,
+      "loss": 0.0628,
+      "num_tokens": 12316984.0,
+      "reward": 0.772705078125,
+      "reward_std": 0.008392134681344032,
+      "rewards//mean": 0.772705078125,
+      "rewards//std": 0.029674729332327843,
+      "step": 1425
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2852,
+      "grad_norm": 0.04022868350148201,
+      "kl": 0.5108750946819782,
+      "learning_rate": 8.213938048432696e-07,
+      "loss": 0.0204,
+      "num_tokens": 12325648.0,
+      "reward": 0.76953125,
+      "reward_std": 0.0,
+      "rewards//mean": 0.76953125,
+      "rewards//std": 0.018466776236891747,
+      "step": 1426
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2854,
+      "grad_norm": 4.986400604248047,
+      "kl": 2.078704100102186,
+      "learning_rate": 8.211506492627318e-07,
+      "loss": 0.0831,
+      "num_tokens": 12334280.0,
+      "reward": 0.74029541015625,
+      "reward_std": 0.008886216208338737,
+      "rewards//mean": 0.74029541015625,
+      "rewards//std": 0.028030389919877052,
+      "step": 1427
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2856,
+      "grad_norm": 3.4062979221343994,
+      "kl": 1.296766459941864,
+      "learning_rate": 8.209073643227509e-07,
+      "loss": 0.0519,
+      "num_tokens": 12342960.0,
+      "reward": 0.75018310546875,
+      "reward_std": 0.004315837286412716,
+      "rewards//mean": 0.75018310546875,
+      "rewards//std": 0.0354849174618721,
+      "step": 1428
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2858,
+      "grad_norm": 6.825105667114258,
+      "kl": 1.2616166323423386,
+      "learning_rate": 8.206639501213219e-07,
+      "loss": 0.0505,
+      "num_tokens": 12351680.0,
+      "reward": 0.81048583984375,
+      "reward_std": 0.0031310291960835457,
+      "rewards//mean": 0.81048583984375,
+      "rewards//std": 0.01588621363043785,
+      "step": 1429
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 126.453125,
+      "epoch": 0.286,
+      "grad_norm": 15.299839973449707,
+      "kl": 5.4106599148362875,
+      "learning_rate": 8.204204067564924e-07,
+      "loss": 0.1848,
+      "num_tokens": 12360205.0,
+      "reward": 0.76239013671875,
+      "reward_std": 0.01022109854966402,
+      "rewards//mean": 0.76239013671875,
+      "rewards//std": 0.021559547632932663,
+      "step": 1430
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2862,
+      "grad_norm": 3.5731098651885986,
+      "kl": 3.9427863769233227,
+      "learning_rate": 8.201767343263611e-07,
+      "loss": 0.1577,
+      "num_tokens": 12368885.0,
+      "reward": 0.730712890625,
+      "reward_std": 0.013175707310438156,
+      "rewards//mean": 0.730712890625,
+      "rewards//std": 0.04189203307032585,
+      "step": 1431
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2864,
+      "grad_norm": 11.419472694396973,
+      "kl": 5.961509704589844,
+      "learning_rate": 8.199329329290796e-07,
+      "loss": 0.2385,
+      "num_tokens": 12377629.0,
+      "reward": 0.7703857421875,
+      "reward_std": 0.01069999486207962,
+      "rewards//mean": 0.7703857421875,
+      "rewards//std": 0.03586537390947342,
+      "step": 1432
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2866,
+      "grad_norm": 5.172885894775391,
+      "kl": 4.274353949353099,
+      "learning_rate": 8.19689002662851e-07,
+      "loss": 0.171,
+      "num_tokens": 12386205.0,
+      "reward": 0.7227783203125,
+      "reward_std": 0.010881026275455952,
+      "rewards//mean": 0.7227783203125,
+      "rewards//std": 0.036551013588905334,
+      "step": 1433
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2868,
+      "grad_norm": 5.536722183227539,
+      "kl": 3.0592558216303587,
+      "learning_rate": 8.194449436259303e-07,
+      "loss": 0.1224,
+      "num_tokens": 12394781.0,
+      "reward": 0.7886962890625,
+      "reward_std": 0.006747500505298376,
+      "rewards//mean": 0.7886962890625,
+      "rewards//std": 0.020047960802912712,
+      "step": 1434
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.287,
+      "grad_norm": 6.103031635284424,
+      "kl": 1.1958512663841248,
+      "learning_rate": 8.192007559166247e-07,
+      "loss": 0.0478,
+      "num_tokens": 12403413.0,
+      "reward": 0.73822021484375,
+      "reward_std": 0.00039866380393505096,
+      "rewards//mean": 0.73822021484375,
+      "rewards//std": 0.02081579715013504,
+      "step": 1435
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2872,
+      "grad_norm": 2.698838710784912,
+      "kl": 2.695367092266679,
+      "learning_rate": 8.189564396332926e-07,
+      "loss": 0.1078,
+      "num_tokens": 12412045.0,
+      "reward": 0.79327392578125,
+      "reward_std": 0.008445193991065025,
+      "rewards//mean": 0.79327392578125,
+      "rewards//std": 0.02185799553990364,
+      "step": 1436
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2874,
+      "grad_norm": 1.4131667613983154,
+      "kl": 1.029098181053996,
+      "learning_rate": 8.187119948743449e-07,
+      "loss": 0.0412,
+      "num_tokens": 12420741.0,
+      "reward": 0.73797607421875,
+      "reward_std": 0.0010980158112943172,
+      "rewards//mean": 0.73797607421875,
+      "rewards//std": 0.01357283629477024,
+      "step": 1437
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2876,
+      "grad_norm": 3.045125722885132,
+      "kl": 3.2402882408350706,
+      "learning_rate": 8.184674217382437e-07,
+      "loss": 0.1296,
+      "num_tokens": 12429421.0,
+      "reward": 0.724609375,
+      "reward_std": 0.008203878998756409,
+      "rewards//mean": 0.724609375,
+      "rewards//std": 0.03579460829496384,
+      "step": 1438
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2878,
+      "grad_norm": 3.918938159942627,
+      "kl": 2.795470768585801,
+      "learning_rate": 8.182227203235031e-07,
+      "loss": 0.1118,
+      "num_tokens": 12438141.0,
+      "reward": 0.76739501953125,
+      "reward_std": 0.008533384650945663,
+      "rewards//mean": 0.76739501953125,
+      "rewards//std": 0.029748542234301567,
+      "step": 1439
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.288,
+      "grad_norm": 3.2863383293151855,
+      "kl": 2.1188965663313866,
+      "learning_rate": 8.179778907286887e-07,
+      "loss": 0.0848,
+      "num_tokens": 12446805.0,
+      "reward": 0.7318115234375,
+      "reward_std": 0.005179004743695259,
+      "rewards//mean": 0.7318115234375,
+      "rewards//std": 0.023242603987455368,
+      "step": 1440
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2882,
+      "grad_norm": 22.631269454956055,
+      "kl": 8.918640416115522,
+      "learning_rate": 8.177329330524181e-07,
+      "loss": 0.3567,
+      "num_tokens": 12455789.0,
+      "reward": 0.6925048828125,
+      "reward_std": 0.013418271206319332,
+      "rewards//mean": 0.6925048828125,
+      "rewards//std": 0.04893512278795242,
+      "step": 1441
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2884,
+      "grad_norm": 5.682319164276123,
+      "kl": 4.445296753197908,
+      "learning_rate": 8.1748784739336e-07,
+      "loss": 0.1778,
+      "num_tokens": 12464421.0,
+      "reward": 0.74005126953125,
+      "reward_std": 0.005198127590119839,
+      "rewards//mean": 0.74005126953125,
+      "rewards//std": 0.02654763124883175,
+      "step": 1442
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2886,
+      "grad_norm": 1.4470422267913818,
+      "kl": 1.0345830917358398,
+      "learning_rate": 8.17242633850235e-07,
+      "loss": 0.0414,
+      "num_tokens": 12473037.0,
+      "reward": 0.771484375,
+      "reward_std": 0.0019523651571944356,
+      "rewards//mean": 0.771484375,
+      "rewards//std": 0.016703827306628227,
+      "step": 1443
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2888,
+      "grad_norm": 2.220828056335449,
+      "kl": 2.345273908227682,
+      "learning_rate": 8.16997292521815e-07,
+      "loss": 0.0938,
+      "num_tokens": 12481677.0,
+      "reward": 0.78546142578125,
+      "reward_std": 0.005727018695324659,
+      "rewards//mean": 0.78546142578125,
+      "rewards//std": 0.026605729013681412,
+      "step": 1444
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.289,
+      "grad_norm": 5.149527549743652,
+      "kl": 4.182993354275823,
+      "learning_rate": 8.167518235069234e-07,
+      "loss": 0.1673,
+      "num_tokens": 12490309.0,
+      "reward": 0.771484375,
+      "reward_std": 0.00904484186321497,
+      "rewards//mean": 0.771484375,
+      "rewards//std": 0.024449070915579796,
+      "step": 1445
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2892,
+      "grad_norm": 1.7928732633590698,
+      "kl": 2.3073074258863926,
+      "learning_rate": 8.165062269044352e-07,
+      "loss": 0.0923,
+      "num_tokens": 12498853.0,
+      "reward": 0.74029541015625,
+      "reward_std": 0.005118072032928467,
+      "rewards//mean": 0.74029541015625,
+      "rewards//std": 0.02886476367712021,
+      "step": 1446
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2894,
+      "grad_norm": 2.6357505321502686,
+      "kl": 1.0002813152968884,
+      "learning_rate": 8.162605028132768e-07,
+      "loss": 0.04,
+      "num_tokens": 12507541.0,
+      "reward": 0.78741455078125,
+      "reward_std": 0.0024336238857358694,
+      "rewards//mean": 0.78741455078125,
+      "rewards//std": 0.01627499610185623,
+      "step": 1447
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2896,
+      "grad_norm": 12.989418983459473,
+      "kl": 6.614552399143577,
+      "learning_rate": 8.160146513324254e-07,
+      "loss": 0.2646,
+      "num_tokens": 12516237.0,
+      "reward": 0.723388671875,
+      "reward_std": 0.008351363241672516,
+      "rewards//mean": 0.723388671875,
+      "rewards//std": 0.02559962496161461,
+      "step": 1448
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2898,
+      "grad_norm": 13.686028480529785,
+      "kl": 2.834360908716917,
+      "learning_rate": 8.157686725609105e-07,
+      "loss": 0.1134,
+      "num_tokens": 12524869.0,
+      "reward": 0.78387451171875,
+      "reward_std": 0.009840108454227448,
+      "rewards//mean": 0.78387451171875,
+      "rewards//std": 0.02367454767227173,
+      "step": 1449
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.29,
+      "grad_norm": 12.314616203308105,
+      "kl": 6.877235503867269,
+      "learning_rate": 8.155225665978118e-07,
+      "loss": 0.2751,
+      "num_tokens": 12533581.0,
+      "reward": 0.74212646484375,
+      "reward_std": 0.00912742130458355,
+      "rewards//mean": 0.74212646484375,
+      "rewards//std": 0.03233553096652031,
+      "step": 1450
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2902,
+      "grad_norm": 3.5310564041137695,
+      "kl": 1.5004207510501146,
+      "learning_rate": 8.152763335422612e-07,
+      "loss": 0.06,
+      "num_tokens": 12542157.0,
+      "reward": 0.77142333984375,
+      "reward_std": 0.006713591516017914,
+      "rewards//mean": 0.77142333984375,
+      "rewards//std": 0.025065330788493156,
+      "step": 1451
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2904,
+      "grad_norm": 3.1590774059295654,
+      "kl": 2.933615291491151,
+      "learning_rate": 8.150299734934412e-07,
+      "loss": 0.1173,
+      "num_tokens": 12550805.0,
+      "reward": 0.74053955078125,
+      "reward_std": 0.0076324716210365295,
+      "rewards//mean": 0.74053955078125,
+      "rewards//std": 0.021951979026198387,
+      "step": 1452
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2906,
+      "grad_norm": 5.115265846252441,
+      "kl": 3.4152608290314674,
+      "learning_rate": 8.147834865505853e-07,
+      "loss": 0.1366,
+      "num_tokens": 12559397.0,
+      "reward": 0.73199462890625,
+      "reward_std": 0.012220550328493118,
+      "rewards//mean": 0.73199462890625,
+      "rewards//std": 0.041102588176727295,
+      "step": 1453
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2908,
+      "grad_norm": 0.9487642645835876,
+      "kl": 1.1078472509980202,
+      "learning_rate": 8.145368728129789e-07,
+      "loss": 0.0443,
+      "num_tokens": 12567949.0,
+      "reward": 0.74945068359375,
+      "reward_std": 0.0013736373512074351,
+      "rewards//mean": 0.74945068359375,
+      "rewards//std": 0.02379508875310421,
+      "step": 1454
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.291,
+      "grad_norm": 1.6968648433685303,
+      "kl": 1.4060956872999668,
+      "learning_rate": 8.142901323799577e-07,
+      "loss": 0.0562,
+      "num_tokens": 12576581.0,
+      "reward": 0.74066162109375,
+      "reward_std": 0.0010767546482384205,
+      "rewards//mean": 0.74066162109375,
+      "rewards//std": 0.026608573272824287,
+      "step": 1455
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2912,
+      "grad_norm": 5.425151824951172,
+      "kl": 2.681792926043272,
+      "learning_rate": 8.140432653509087e-07,
+      "loss": 0.1073,
+      "num_tokens": 12585237.0,
+      "reward": 0.76397705078125,
+      "reward_std": 0.005783388391137123,
+      "rewards//mean": 0.76397705078125,
+      "rewards//std": 0.028295380994677544,
+      "step": 1456
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2914,
+      "grad_norm": 3.9536635875701904,
+      "kl": 1.6290787030011415,
+      "learning_rate": 8.1379627182527e-07,
+      "loss": 0.0652,
+      "num_tokens": 12593853.0,
+      "reward": 0.75933837890625,
+      "reward_std": 0.0020354087464511395,
+      "rewards//mean": 0.75933837890625,
+      "rewards//std": 0.03408672288060188,
+      "step": 1457
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2916,
+      "grad_norm": 18.338483810424805,
+      "kl": 3.4971011132001877,
+      "learning_rate": 8.135491519025306e-07,
+      "loss": 0.1399,
+      "num_tokens": 12602573.0,
+      "reward": 0.75213623046875,
+      "reward_std": 0.005087354686111212,
+      "rewards//mean": 0.75213623046875,
+      "rewards//std": 0.0330573134124279,
+      "step": 1458
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2918,
+      "grad_norm": 3.185584306716919,
+      "kl": 4.476410383358598,
+      "learning_rate": 8.133019056822302e-07,
+      "loss": 0.1791,
+      "num_tokens": 12611237.0,
+      "reward": 0.76300048828125,
+      "reward_std": 0.006838707718998194,
+      "rewards//mean": 0.76300048828125,
+      "rewards//std": 0.026199806481599808,
+      "step": 1459
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.292,
+      "grad_norm": 5.106291770935059,
+      "kl": 3.9950185026973486,
+      "learning_rate": 8.130545332639597e-07,
+      "loss": 0.1598,
+      "num_tokens": 12619877.0,
+      "reward": 0.74127197265625,
+      "reward_std": 0.00920344889163971,
+      "rewards//mean": 0.74127197265625,
+      "rewards//std": 0.02624310366809368,
+      "step": 1460
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2922,
+      "grad_norm": 22.210561752319336,
+      "kl": 6.73808372579515,
+      "learning_rate": 8.128070347473608e-07,
+      "loss": 0.2695,
+      "num_tokens": 12628621.0,
+      "reward": 0.77655029296875,
+      "reward_std": 0.007952040061354637,
+      "rewards//mean": 0.77655029296875,
+      "rewards//std": 0.030759258195757866,
+      "step": 1461
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2924,
+      "grad_norm": 28.265748977661133,
+      "kl": 5.9607279896736145,
+      "learning_rate": 8.125594102321255e-07,
+      "loss": 0.2384,
+      "num_tokens": 12637245.0,
+      "reward": 0.77655029296875,
+      "reward_std": 0.00983298197388649,
+      "rewards//mean": 0.77655029296875,
+      "rewards//std": 0.03253481537103653,
+      "step": 1462
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2926,
+      "grad_norm": 13.571913719177246,
+      "kl": 3.088777916505933,
+      "learning_rate": 8.123116598179971e-07,
+      "loss": 0.1236,
+      "num_tokens": 12645997.0,
+      "reward": 0.759765625,
+      "reward_std": 0.011301582679152489,
+      "rewards//mean": 0.759765625,
+      "rewards//std": 0.03557742014527321,
+      "step": 1463
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2928,
+      "grad_norm": 53.40056610107422,
+      "kl": 8.706040035933256,
+      "learning_rate": 8.120637836047697e-07,
+      "loss": 0.3482,
+      "num_tokens": 12654605.0,
+      "reward": 0.744384765625,
+      "reward_std": 0.012703519314527512,
+      "rewards//mean": 0.744384765625,
+      "rewards//std": 0.03117142803966999,
+      "step": 1464
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.293,
+      "grad_norm": 38.953224182128906,
+      "kl": 6.153975997120142,
+      "learning_rate": 8.118157816922874e-07,
+      "loss": 0.2462,
+      "num_tokens": 12663277.0,
+      "reward": 0.725830078125,
+      "reward_std": 0.009198610670864582,
+      "rewards//mean": 0.725830078125,
+      "rewards//std": 0.0242488831281662,
+      "step": 1465
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.078125,
+      "epoch": 0.2932,
+      "grad_norm": 20.69542694091797,
+      "kl": 7.51901463791728,
+      "learning_rate": 8.115676541804455e-07,
+      "loss": 0.2882,
+      "num_tokens": 12671834.0,
+      "reward": 0.72149658203125,
+      "reward_std": 0.01875142753124237,
+      "rewards//mean": 0.72149658203125,
+      "rewards//std": 0.042692676186561584,
+      "step": 1466
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2934,
+      "grad_norm": 14.31985855102539,
+      "kl": 6.497452400624752,
+      "learning_rate": 8.113194011691899e-07,
+      "loss": 0.2599,
+      "num_tokens": 12680426.0,
+      "reward": 0.750244140625,
+      "reward_std": 0.012425040826201439,
+      "rewards//mean": 0.750244140625,
+      "rewards//std": 0.031878259032964706,
+      "step": 1467
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2936,
+      "grad_norm": 7.346076011657715,
+      "kl": 2.856573548167944,
+      "learning_rate": 8.110710227585167e-07,
+      "loss": 0.1143,
+      "num_tokens": 12689010.0,
+      "reward": 0.78509521484375,
+      "reward_std": 0.008657204918563366,
+      "rewards//mean": 0.78509521484375,
+      "rewards//std": 0.01946905069053173,
+      "step": 1468
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2938,
+      "grad_norm": 16.012229919433594,
+      "kl": 1.4431650210171938,
+      "learning_rate": 8.108225190484726e-07,
+      "loss": 0.0577,
+      "num_tokens": 12697682.0,
+      "reward": 0.73443603515625,
+      "reward_std": 0.004821329843252897,
+      "rewards//mean": 0.73443603515625,
+      "rewards//std": 0.03247007727622986,
+      "step": 1469
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.294,
+      "grad_norm": 7.685550689697266,
+      "kl": 4.789723701775074,
+      "learning_rate": 8.105738901391551e-07,
+      "loss": 0.1916,
+      "num_tokens": 12706426.0,
+      "reward": 0.79119873046875,
+      "reward_std": 0.016323965042829514,
+      "rewards//mean": 0.79119873046875,
+      "rewards//std": 0.031148534268140793,
+      "step": 1470
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2942,
+      "grad_norm": 13.609522819519043,
+      "kl": 4.810040263459086,
+      "learning_rate": 8.103251361307118e-07,
+      "loss": 0.1924,
+      "num_tokens": 12715122.0,
+      "reward": 0.74664306640625,
+      "reward_std": 0.0036240937188267708,
+      "rewards//mean": 0.74664306640625,
+      "rewards//std": 0.030583063140511513,
+      "step": 1471
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2944,
+      "grad_norm": 15.771655082702637,
+      "kl": 3.396984687075019,
+      "learning_rate": 8.100762571233408e-07,
+      "loss": 0.1359,
+      "num_tokens": 12723730.0,
+      "reward": 0.77655029296875,
+      "reward_std": 0.008351262658834457,
+      "rewards//mean": 0.77655029296875,
+      "rewards//std": 0.021566567942500114,
+      "step": 1472
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2946,
+      "grad_norm": 5.584359169006348,
+      "kl": 2.603633690625429,
+      "learning_rate": 8.098272532172905e-07,
+      "loss": 0.1041,
+      "num_tokens": 12732546.0,
+      "reward": 0.75225830078125,
+      "reward_std": 0.006755417678505182,
+      "rewards//mean": 0.75225830078125,
+      "rewards//std": 0.03003114089369774,
+      "step": 1473
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2948,
+      "grad_norm": 7.366146564483643,
+      "kl": 3.0774579513818026,
+      "learning_rate": 8.095781245128597e-07,
+      "loss": 0.1231,
+      "num_tokens": 12741266.0,
+      "reward": 0.752197265625,
+      "reward_std": 0.015443527139723301,
+      "rewards//mean": 0.752197265625,
+      "rewards//std": 0.037492476403713226,
+      "step": 1474
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.295,
+      "grad_norm": 4.636621475219727,
+      "kl": 3.1825755424797535,
+      "learning_rate": 8.093288711103971e-07,
+      "loss": 0.1273,
+      "num_tokens": 12749826.0,
+      "reward": 0.75592041015625,
+      "reward_std": 0.005983960349112749,
+      "rewards//mean": 0.75592041015625,
+      "rewards//std": 0.028881540521979332,
+      "step": 1475
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2952,
+      "grad_norm": 11.739696502685547,
+      "kl": 2.0741084031760693,
+      "learning_rate": 8.090794931103026e-07,
+      "loss": 0.083,
+      "num_tokens": 12758434.0,
+      "reward": 0.760498046875,
+      "reward_std": 0.006874276790767908,
+      "rewards//mean": 0.760498046875,
+      "rewards//std": 0.023559898138046265,
+      "step": 1476
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2954,
+      "grad_norm": 6.742212772369385,
+      "kl": 3.330030683428049,
+      "learning_rate": 8.08829990613025e-07,
+      "loss": 0.1332,
+      "num_tokens": 12767034.0,
+      "reward": 0.75030517578125,
+      "reward_std": 0.00829287339001894,
+      "rewards//mean": 0.75030517578125,
+      "rewards//std": 0.03372015058994293,
+      "step": 1477
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2956,
+      "grad_norm": 7.87825345993042,
+      "kl": 3.7539873644709587,
+      "learning_rate": 8.085803637190643e-07,
+      "loss": 0.1502,
+      "num_tokens": 12775674.0,
+      "reward": 0.7900390625,
+      "reward_std": 0.01460934802889824,
+      "rewards//mean": 0.7900390625,
+      "rewards//std": 0.03276345133781433,
+      "step": 1478
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2958,
+      "grad_norm": 3.2344954013824463,
+      "kl": 3.3349881507456303,
+      "learning_rate": 8.083306125289697e-07,
+      "loss": 0.1334,
+      "num_tokens": 12784202.0,
+      "reward": 0.75860595703125,
+      "reward_std": 0.010151240974664688,
+      "rewards//mean": 0.75860595703125,
+      "rewards//std": 0.034190040081739426,
+      "step": 1479
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.296,
+      "grad_norm": 7.749207496643066,
+      "kl": 1.7467486821115017,
+      "learning_rate": 8.080807371433414e-07,
+      "loss": 0.0699,
+      "num_tokens": 12792810.0,
+      "reward": 0.76861572265625,
+      "reward_std": 0.009853655472397804,
+      "rewards//mean": 0.76861572265625,
+      "rewards//std": 0.022258087992668152,
+      "step": 1480
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2962,
+      "grad_norm": 5.620584487915039,
+      "kl": 2.2123597227036953,
+      "learning_rate": 8.07830737662829e-07,
+      "loss": 0.0885,
+      "num_tokens": 12801482.0,
+      "reward": 0.74835205078125,
+      "reward_std": 0.004649493843317032,
+      "rewards//mean": 0.74835205078125,
+      "rewards//std": 0.025198446586728096,
+      "step": 1481
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2964,
+      "grad_norm": 3.8430168628692627,
+      "kl": 3.042467314749956,
+      "learning_rate": 8.075806141881325e-07,
+      "loss": 0.1217,
+      "num_tokens": 12810218.0,
+      "reward": 0.7652587890625,
+      "reward_std": 0.011134494096040726,
+      "rewards//mean": 0.7652587890625,
+      "rewards//std": 0.0349019318819046,
+      "step": 1482
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2966,
+      "grad_norm": 16.194698333740234,
+      "kl": 1.9202016908675432,
+      "learning_rate": 8.073303668200011e-07,
+      "loss": 0.0768,
+      "num_tokens": 12818898.0,
+      "reward": 0.77398681640625,
+      "reward_std": 0.006984813138842583,
+      "rewards//mean": 0.77398681640625,
+      "rewards//std": 0.02519904635846615,
+      "step": 1483
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.140625,
+      "epoch": 0.2968,
+      "grad_norm": 15.359682083129883,
+      "kl": 3.4303321093320847,
+      "learning_rate": 8.070799956592349e-07,
+      "loss": 0.1215,
+      "num_tokens": 12827403.0,
+      "reward": 0.74493408203125,
+      "reward_std": 0.00778286624699831,
+      "rewards//mean": 0.74493408203125,
+      "rewards//std": 0.025569375604391098,
+      "step": 1484
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.297,
+      "grad_norm": 15.314162254333496,
+      "kl": 0.5339925810694695,
+      "learning_rate": 8.06829500806683e-07,
+      "loss": 0.0214,
+      "num_tokens": 12836035.0,
+      "reward": 0.7694091796875,
+      "reward_std": 0.0026868688873946667,
+      "rewards//mean": 0.7694091796875,
+      "rewards//std": 0.021349729970097542,
+      "step": 1485
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2972,
+      "grad_norm": 13.411092758178711,
+      "kl": 6.477550081908703,
+      "learning_rate": 8.06578882363245e-07,
+      "loss": 0.2591,
+      "num_tokens": 12844763.0,
+      "reward": 0.7552490234375,
+      "reward_std": 0.019006986171007156,
+      "rewards//mean": 0.7552490234375,
+      "rewards//std": 0.03357556834816933,
+      "step": 1486
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2974,
+      "grad_norm": 9.768086433410645,
+      "kl": 1.3256912045180798,
+      "learning_rate": 8.063281404298699e-07,
+      "loss": 0.053,
+      "num_tokens": 12853435.0,
+      "reward": 0.7830810546875,
+      "reward_std": 0.006776880472898483,
+      "rewards//mean": 0.7830810546875,
+      "rewards//std": 0.028202474117279053,
+      "step": 1487
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2976,
+      "grad_norm": 24.31314468383789,
+      "kl": 3.6333701610565186,
+      "learning_rate": 8.060772751075562e-07,
+      "loss": 0.1453,
+      "num_tokens": 12862083.0,
+      "reward": 0.7589111328125,
+      "reward_std": 0.012979390099644661,
+      "rewards//mean": 0.7589111328125,
+      "rewards//std": 0.02355572022497654,
+      "step": 1488
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2978,
+      "grad_norm": 11.574331283569336,
+      "kl": 1.9029613602906466,
+      "learning_rate": 8.058262864973528e-07,
+      "loss": 0.0761,
+      "num_tokens": 12870739.0,
+      "reward": 0.744873046875,
+      "reward_std": 0.00930815003812313,
+      "rewards//mean": 0.744873046875,
+      "rewards//std": 0.03298356384038925,
+      "step": 1489
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.298,
+      "grad_norm": 24.865310668945312,
+      "kl": 3.1038744542747736,
+      "learning_rate": 8.055751747003579e-07,
+      "loss": 0.1242,
+      "num_tokens": 12879339.0,
+      "reward": 0.73614501953125,
+      "reward_std": 0.01602857932448387,
+      "rewards//mean": 0.73614501953125,
+      "rewards//std": 0.033501263707876205,
+      "step": 1490
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2982,
+      "grad_norm": 26.77971839904785,
+      "kl": 5.77071532048285,
+      "learning_rate": 8.053239398177191e-07,
+      "loss": 0.2308,
+      "num_tokens": 12887931.0,
+      "reward": 0.746826171875,
+      "reward_std": 0.0146062932908535,
+      "rewards//mean": 0.746826171875,
+      "rewards//std": 0.033652350306510925,
+      "step": 1491
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2984,
+      "grad_norm": 11.379695892333984,
+      "kl": 3.4608402214944363,
+      "learning_rate": 8.050725819506339e-07,
+      "loss": 0.1384,
+      "num_tokens": 12896563.0,
+      "reward": 0.753173828125,
+      "reward_std": 0.010060532949864864,
+      "rewards//mean": 0.753173828125,
+      "rewards//std": 0.029715511947870255,
+      "step": 1492
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2986,
+      "grad_norm": 17.708709716796875,
+      "kl": 3.076208494603634,
+      "learning_rate": 8.048211012003489e-07,
+      "loss": 0.123,
+      "num_tokens": 12905275.0,
+      "reward": 0.74200439453125,
+      "reward_std": 0.010404210537672043,
+      "rewards//mean": 0.74200439453125,
+      "rewards//std": 0.02500183694064617,
+      "step": 1493
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2988,
+      "grad_norm": 11.853999137878418,
+      "kl": 1.472416803240776,
+      "learning_rate": 8.045694976681612e-07,
+      "loss": 0.0589,
+      "num_tokens": 12913971.0,
+      "reward": 0.74090576171875,
+      "reward_std": 0.004838705062866211,
+      "rewards//mean": 0.74090576171875,
+      "rewards//std": 0.03022608533501625,
+      "step": 1494
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.299,
+      "grad_norm": 14.601651191711426,
+      "kl": 1.9102000687271357,
+      "learning_rate": 8.043177714554159e-07,
+      "loss": 0.0764,
+      "num_tokens": 12922619.0,
+      "reward": 0.77532958984375,
+      "reward_std": 0.007799575570970774,
+      "rewards//mean": 0.77532958984375,
+      "rewards//std": 0.027081118896603584,
+      "step": 1495
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2992,
+      "grad_norm": 8.707847595214844,
+      "kl": 1.6990231294184923,
+      "learning_rate": 8.04065922663509e-07,
+      "loss": 0.068,
+      "num_tokens": 12931323.0,
+      "reward": 0.79119873046875,
+      "reward_std": 0.007749687880277634,
+      "rewards//mean": 0.79119873046875,
+      "rewards//std": 0.031241703778505325,
+      "step": 1496
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2994,
+      "grad_norm": 13.480512619018555,
+      "kl": 4.101980570703745,
+      "learning_rate": 8.038139513938845e-07,
+      "loss": 0.1641,
+      "num_tokens": 12940035.0,
+      "reward": 0.74798583984375,
+      "reward_std": 0.011941012926399708,
+      "rewards//mean": 0.74798583984375,
+      "rewards//std": 0.031703922897577286,
+      "step": 1497
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2996,
+      "grad_norm": 6.592166423797607,
+      "kl": 4.812883865088224,
+      "learning_rate": 8.035618577480369e-07,
+      "loss": 0.1925,
+      "num_tokens": 12948675.0,
+      "reward": 0.78741455078125,
+      "reward_std": 0.016344359144568443,
+      "rewards//mean": 0.78741455078125,
+      "rewards//std": 0.026280570775270462,
+      "step": 1498
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.2998,
+      "grad_norm": 4.115764617919922,
+      "kl": 3.6408120151609182,
+      "learning_rate": 8.033096418275092e-07,
+      "loss": 0.1456,
+      "num_tokens": 12957371.0,
+      "reward": 0.72442626953125,
+      "reward_std": 0.01087966002523899,
+      "rewards//mean": 0.72442626953125,
+      "rewards//std": 0.042329467833042145,
+      "step": 1499
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3,
+      "grad_norm": 8.221807479858398,
+      "kl": 1.7563366871327162,
+      "learning_rate": 8.030573037338941e-07,
+      "loss": 0.0703,
+      "num_tokens": 12966027.0,
+      "reward": 0.77227783203125,
+      "reward_std": 0.008386017754673958,
+      "rewards//mean": 0.77227783203125,
+      "rewards//std": 0.025786321610212326,
+      "step": 1500
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3002,
+      "grad_norm": 5.701700210571289,
+      "kl": 0.5587639026343822,
+      "learning_rate": 8.028048435688333e-07,
+      "loss": 0.0224,
+      "num_tokens": 12974603.0,
+      "reward": 0.79888916015625,
+      "reward_std": 0.001270482549443841,
+      "rewards//mean": 0.79888916015625,
+      "rewards//std": 0.019586877897381783,
+      "step": 1501
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3004,
+      "grad_norm": 3.7195465564727783,
+      "kl": 2.6717438362538815,
+      "learning_rate": 8.025522614340177e-07,
+      "loss": 0.1069,
+      "num_tokens": 12983131.0,
+      "reward": 0.74420166015625,
+      "reward_std": 0.010117185302078724,
+      "rewards//mean": 0.74420166015625,
+      "rewards//std": 0.03271160647273064,
+      "step": 1502
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3006,
+      "grad_norm": 3.6980080604553223,
+      "kl": 3.450033061206341,
+      "learning_rate": 8.022995574311875e-07,
+      "loss": 0.138,
+      "num_tokens": 12991811.0,
+      "reward": 0.7396240234375,
+      "reward_std": 0.010044010356068611,
+      "rewards//mean": 0.7396240234375,
+      "rewards//std": 0.025602877140045166,
+      "step": 1503
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3008,
+      "grad_norm": 2.8952481746673584,
+      "kl": 0.790763495489955,
+      "learning_rate": 8.020467316621316e-07,
+      "loss": 0.0316,
+      "num_tokens": 13000387.0,
+      "reward": 0.78289794921875,
+      "reward_std": 0.001645436743274331,
+      "rewards//mean": 0.78289794921875,
+      "rewards//std": 0.022193383425474167,
+      "step": 1504
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.301,
+      "grad_norm": 2.669785976409912,
+      "kl": 4.1806445978581905,
+      "learning_rate": 8.017937842286882e-07,
+      "loss": 0.1672,
+      "num_tokens": 13009059.0,
+      "reward": 0.75775146484375,
+      "reward_std": 0.013202082365751266,
+      "rewards//mean": 0.75775146484375,
+      "rewards//std": 0.028843779116868973,
+      "step": 1505
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3012,
+      "grad_norm": 2.4568898677825928,
+      "kl": 0.747019100934267,
+      "learning_rate": 8.015407152327447e-07,
+      "loss": 0.0299,
+      "num_tokens": 13017659.0,
+      "reward": 0.74664306640625,
+      "reward_std": 0.0028939624316990376,
+      "rewards//mean": 0.74664306640625,
+      "rewards//std": 0.025218263268470764,
+      "step": 1506
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3014,
+      "grad_norm": 1.8890511989593506,
+      "kl": 1.1498551853001118,
+      "learning_rate": 8.012875247762372e-07,
+      "loss": 0.046,
+      "num_tokens": 13026251.0,
+      "reward": 0.78240966796875,
+      "reward_std": 0.002637900412082672,
+      "rewards//mean": 0.78240966796875,
+      "rewards//std": 0.019828831776976585,
+      "step": 1507
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3016,
+      "grad_norm": 4.576070308685303,
+      "kl": 1.594864496961236,
+      "learning_rate": 8.010342129611507e-07,
+      "loss": 0.0638,
+      "num_tokens": 13034867.0,
+      "reward": 0.74298095703125,
+      "reward_std": 0.0036703976802527905,
+      "rewards//mean": 0.74298095703125,
+      "rewards//std": 0.023085355758666992,
+      "step": 1508
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3018,
+      "grad_norm": 3.2699155807495117,
+      "kl": 1.748958621174097,
+      "learning_rate": 8.007807798895193e-07,
+      "loss": 0.07,
+      "num_tokens": 13043611.0,
+      "reward": 0.74957275390625,
+      "reward_std": 0.004528774879872799,
+      "rewards//mean": 0.74957275390625,
+      "rewards//std": 0.026402974501252174,
+      "step": 1509
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.302,
+      "grad_norm": 4.456327438354492,
+      "kl": 0.8866036608815193,
+      "learning_rate": 8.005272256634257e-07,
+      "loss": 0.0355,
+      "num_tokens": 13052251.0,
+      "reward": 0.75701904296875,
+      "reward_std": 0.004231567494571209,
+      "rewards//mean": 0.75701904296875,
+      "rewards//std": 0.027980120852589607,
+      "step": 1510
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3022,
+      "grad_norm": 1.179195523262024,
+      "kl": 1.5088980309665203,
+      "learning_rate": 8.002735503850015e-07,
+      "loss": 0.0604,
+      "num_tokens": 13060931.0,
+      "reward": 0.77215576171875,
+      "reward_std": 0.002582917455583811,
+      "rewards//mean": 0.77215576171875,
+      "rewards//std": 0.024725429713726044,
+      "step": 1511
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3024,
+      "grad_norm": 5.7419586181640625,
+      "kl": 2.186433929949999,
+      "learning_rate": 8.000197541564271e-07,
+      "loss": 0.0875,
+      "num_tokens": 13069595.0,
+      "reward": 0.7386474609375,
+      "reward_std": 0.003107402939349413,
+      "rewards//mean": 0.7386474609375,
+      "rewards//std": 0.030700454488396645,
+      "step": 1512
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3026,
+      "grad_norm": 3.0364153385162354,
+      "kl": 2.8216484375298023,
+      "learning_rate": 7.997658370799316e-07,
+      "loss": 0.1129,
+      "num_tokens": 13078187.0,
+      "reward": 0.7657470703125,
+      "reward_std": 0.007853852584958076,
+      "rewards//mean": 0.7657470703125,
+      "rewards//std": 0.022173339501023293,
+      "step": 1513
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3028,
+      "grad_norm": 7.140042781829834,
+      "kl": 0.7660270165652037,
+      "learning_rate": 7.995117992577928e-07,
+      "loss": 0.0306,
+      "num_tokens": 13086763.0,
+      "reward": 0.7581787109375,
+      "reward_std": 0.004557499662041664,
+      "rewards//mean": 0.7581787109375,
+      "rewards//std": 0.019926784560084343,
+      "step": 1514
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.303,
+      "grad_norm": 1.8625723123550415,
+      "kl": 0.9544672500342131,
+      "learning_rate": 7.992576407923372e-07,
+      "loss": 0.0382,
+      "num_tokens": 13095403.0,
+      "reward": 0.75994873046875,
+      "reward_std": 0.002039259532466531,
+      "rewards//mean": 0.75994873046875,
+      "rewards//std": 0.024058910086750984,
+      "step": 1515
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3032,
+      "grad_norm": 3.971250295639038,
+      "kl": 2.17875063046813,
+      "learning_rate": 7.990033617859395e-07,
+      "loss": 0.0872,
+      "num_tokens": 13103963.0,
+      "reward": 0.74371337890625,
+      "reward_std": 0.0056187184527516365,
+      "rewards//mean": 0.74371337890625,
+      "rewards//std": 0.025071369484066963,
+      "step": 1516
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3034,
+      "grad_norm": 8.928879737854004,
+      "kl": 2.4854452945291996,
+      "learning_rate": 7.987489623410235e-07,
+      "loss": 0.0994,
+      "num_tokens": 13112563.0,
+      "reward": 0.74102783203125,
+      "reward_std": 0.0031742537394165993,
+      "rewards//mean": 0.74102783203125,
+      "rewards//std": 0.02809673734009266,
+      "step": 1517
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3036,
+      "grad_norm": 1.7843027114868164,
+      "kl": 1.4118167273700237,
+      "learning_rate": 7.984944425600613e-07,
+      "loss": 0.0565,
+      "num_tokens": 13121155.0,
+      "reward": 0.76629638671875,
+      "reward_std": 0.005580378696322441,
+      "rewards//mean": 0.76629638671875,
+      "rewards//std": 0.025841444730758667,
+      "step": 1518
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3038,
+      "grad_norm": 4.163170337677002,
+      "kl": 1.2572334352880716,
+      "learning_rate": 7.982398025455732e-07,
+      "loss": 0.0503,
+      "num_tokens": 13129923.0,
+      "reward": 0.78656005859375,
+      "reward_std": 0.0058036986738443375,
+      "rewards//mean": 0.78656005859375,
+      "rewards//std": 0.02161354385316372,
+      "step": 1519
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.304,
+      "grad_norm": 1.428679347038269,
+      "kl": 0.9906759764999151,
+      "learning_rate": 7.979850424001282e-07,
+      "loss": 0.0396,
+      "num_tokens": 13138555.0,
+      "reward": 0.76422119140625,
+      "reward_std": 0.0012864710297435522,
+      "rewards//mean": 0.76422119140625,
+      "rewards//std": 0.017817514017224312,
+      "step": 1520
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3042,
+      "grad_norm": 3.2875068187713623,
+      "kl": 1.7363167777657509,
+      "learning_rate": 7.97730162226344e-07,
+      "loss": 0.0695,
+      "num_tokens": 13147267.0,
+      "reward": 0.75335693359375,
+      "reward_std": 0.0042083170264959335,
+      "rewards//mean": 0.75335693359375,
+      "rewards//std": 0.021697426214814186,
+      "step": 1521
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3044,
+      "grad_norm": 9.200188636779785,
+      "kl": 6.625841850414872,
+      "learning_rate": 7.974751621268858e-07,
+      "loss": 0.265,
+      "num_tokens": 13155963.0,
+      "reward": 0.76275634765625,
+      "reward_std": 0.011108377017080784,
+      "rewards//mean": 0.76275634765625,
+      "rewards//std": 0.033315036445856094,
+      "step": 1522
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3046,
+      "grad_norm": 1.9781839847564697,
+      "kl": 1.9428846444934607,
+      "learning_rate": 7.972200422044682e-07,
+      "loss": 0.0777,
+      "num_tokens": 13164587.0,
+      "reward": 0.75537109375,
+      "reward_std": 0.005958822090178728,
+      "rewards//mean": 0.75537109375,
+      "rewards//std": 0.026852156966924667,
+      "step": 1523
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3048,
+      "grad_norm": 6.879197120666504,
+      "kl": 2.205456007272005,
+      "learning_rate": 7.969648025618529e-07,
+      "loss": 0.0882,
+      "num_tokens": 13173219.0,
+      "reward": 0.7874755859375,
+      "reward_std": 0.008457712829113007,
+      "rewards//mean": 0.7874755859375,
+      "rewards//std": 0.02719470113515854,
+      "step": 1524
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.305,
+      "grad_norm": 6.487525939941406,
+      "kl": 1.2132704816758633,
+      "learning_rate": 7.967094433018508e-07,
+      "loss": 0.0485,
+      "num_tokens": 13181795.0,
+      "reward": 0.765380859375,
+      "reward_std": 0.0031874789856374264,
+      "rewards//mean": 0.765380859375,
+      "rewards//std": 0.015899665653705597,
+      "step": 1525
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3052,
+      "grad_norm": 3.5539145469665527,
+      "kl": 3.9248960874974728,
+      "learning_rate": 7.964539645273202e-07,
+      "loss": 0.157,
+      "num_tokens": 13190347.0,
+      "reward": 0.7828369140625,
+      "reward_std": 0.00914781354367733,
+      "rewards//mean": 0.7828369140625,
+      "rewards//std": 0.021572621539235115,
+      "step": 1526
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3054,
+      "grad_norm": 5.06986141204834,
+      "kl": 4.15170344337821,
+      "learning_rate": 7.961983663411684e-07,
+      "loss": 0.1661,
+      "num_tokens": 13198987.0,
+      "reward": 0.7686767578125,
+      "reward_std": 0.007378567010164261,
+      "rewards//mean": 0.7686767578125,
+      "rewards//std": 0.030524414032697678,
+      "step": 1527
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3056,
+      "grad_norm": 1.2802878618240356,
+      "kl": 1.1794881578534842,
+      "learning_rate": 7.959426488463499e-07,
+      "loss": 0.0472,
+      "num_tokens": 13207627.0,
+      "reward": 0.74786376953125,
+      "reward_std": 0.0021249987185001373,
+      "rewards//mean": 0.74786376953125,
+      "rewards//std": 0.01725907437503338,
+      "step": 1528
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3058,
+      "grad_norm": 8.874640464782715,
+      "kl": 4.919972432777286,
+      "learning_rate": 7.956868121458677e-07,
+      "loss": 0.1968,
+      "num_tokens": 13216195.0,
+      "reward": 0.7379150390625,
+      "reward_std": 0.007485592737793922,
+      "rewards//mean": 0.7379150390625,
+      "rewards//std": 0.025588683784008026,
+      "step": 1529
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.306,
+      "grad_norm": 12.111811637878418,
+      "kl": 3.570584613829851,
+      "learning_rate": 7.954308563427732e-07,
+      "loss": 0.1428,
+      "num_tokens": 13224811.0,
+      "reward": 0.76019287109375,
+      "reward_std": 0.006022367626428604,
+      "rewards//mean": 0.76019287109375,
+      "rewards//std": 0.0230584554374218,
+      "step": 1530
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3062,
+      "grad_norm": 1.4922840595245361,
+      "kl": 2.025520922616124,
+      "learning_rate": 7.951747815401649e-07,
+      "loss": 0.081,
+      "num_tokens": 13233507.0,
+      "reward": 0.7764892578125,
+      "reward_std": 0.0044884709641337395,
+      "rewards//mean": 0.7764892578125,
+      "rewards//std": 0.031004654243588448,
+      "step": 1531
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3064,
+      "grad_norm": 1.084610939025879,
+      "kl": 1.5145159009844065,
+      "learning_rate": 7.949185878411899e-07,
+      "loss": 0.0606,
+      "num_tokens": 13242139.0,
+      "reward": 0.80169677734375,
+      "reward_std": 0.004900915548205376,
+      "rewards//mean": 0.80169677734375,
+      "rewards//std": 0.024868888780474663,
+      "step": 1532
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3066,
+      "grad_norm": 4.220397472381592,
+      "kl": 4.190284017473459,
+      "learning_rate": 7.946622753490432e-07,
+      "loss": 0.1676,
+      "num_tokens": 13250763.0,
+      "reward": 0.7738037109375,
+      "reward_std": 0.011110790073871613,
+      "rewards//mean": 0.7738037109375,
+      "rewards//std": 0.03179338201880455,
+      "step": 1533
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3068,
+      "grad_norm": 3.8574328422546387,
+      "kl": 3.023826414719224,
+      "learning_rate": 7.94405844166967e-07,
+      "loss": 0.121,
+      "num_tokens": 13259403.0,
+      "reward": 0.7447509765625,
+      "reward_std": 0.006219988223165274,
+      "rewards//mean": 0.7447509765625,
+      "rewards//std": 0.03421863913536072,
+      "step": 1534
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.307,
+      "grad_norm": 7.903994083404541,
+      "kl": 4.962822580710053,
+      "learning_rate": 7.941492943982521e-07,
+      "loss": 0.1985,
+      "num_tokens": 13268131.0,
+      "reward": 0.74761962890625,
+      "reward_std": 0.009986553341150284,
+      "rewards//mean": 0.74761962890625,
+      "rewards//std": 0.032731037586927414,
+      "step": 1535
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3072,
+      "grad_norm": 12.15711498260498,
+      "kl": 4.283036205917597,
+      "learning_rate": 7.938926261462365e-07,
+      "loss": 0.1713,
+      "num_tokens": 13276771.0,
+      "reward": 0.747314453125,
+      "reward_std": 0.00761657627299428,
+      "rewards//mean": 0.747314453125,
+      "rewards//std": 0.02559962496161461,
+      "step": 1536
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3074,
+      "grad_norm": 1.8574138879776,
+      "kl": 2.006930122151971,
+      "learning_rate": 7.936358395143063e-07,
+      "loss": 0.0803,
+      "num_tokens": 13285411.0,
+      "reward": 0.7672119140625,
+      "reward_std": 0.005736600141972303,
+      "rewards//mean": 0.7672119140625,
+      "rewards//std": 0.01953396387398243,
+      "step": 1537
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3076,
+      "grad_norm": 9.763838768005371,
+      "kl": 6.778828650712967,
+      "learning_rate": 7.93378934605895e-07,
+      "loss": 0.2712,
+      "num_tokens": 13294099.0,
+      "reward": 0.76153564453125,
+      "reward_std": 0.012372822500765324,
+      "rewards//mean": 0.76153564453125,
+      "rewards//std": 0.023760074749588966,
+      "step": 1538
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3078,
+      "grad_norm": 2.0672945976257324,
+      "kl": 1.9476182386279106,
+      "learning_rate": 7.93121911524484e-07,
+      "loss": 0.0779,
+      "num_tokens": 13302675.0,
+      "reward": 0.7374267578125,
+      "reward_std": 0.003018442541360855,
+      "rewards//mean": 0.7374267578125,
+      "rewards//std": 0.028429146856069565,
+      "step": 1539
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.308,
+      "grad_norm": 2.7123074531555176,
+      "kl": 2.072697926312685,
+      "learning_rate": 7.928647703736023e-07,
+      "loss": 0.0829,
+      "num_tokens": 13311403.0,
+      "reward": 0.7547607421875,
+      "reward_std": 0.004106986802071333,
+      "rewards//mean": 0.7547607421875,
+      "rewards//std": 0.02630511112511158,
+      "step": 1540
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3082,
+      "grad_norm": 1.635036587715149,
+      "kl": 0.9788522999733686,
+      "learning_rate": 7.926075112568258e-07,
+      "loss": 0.0392,
+      "num_tokens": 13319987.0,
+      "reward": 0.75177001953125,
+      "reward_std": 0.0025895023718476295,
+      "rewards//mean": 0.75177001953125,
+      "rewards//std": 0.02867904119193554,
+      "step": 1541
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3084,
+      "grad_norm": 2.1284656524658203,
+      "kl": 1.0090431105345488,
+      "learning_rate": 7.923501342777787e-07,
+      "loss": 0.0404,
+      "num_tokens": 13328547.0,
+      "reward": 0.73284912109375,
+      "reward_std": 0.002874338533729315,
+      "rewards//mean": 0.73284912109375,
+      "rewards//std": 0.035764940083026886,
+      "step": 1542
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3086,
+      "grad_norm": 2.770582914352417,
+      "kl": 3.5458790939301252,
+      "learning_rate": 7.920926395401326e-07,
+      "loss": 0.1418,
+      "num_tokens": 13337123.0,
+      "reward": 0.7674560546875,
+      "reward_std": 0.007663349155336618,
+      "rewards//mean": 0.7674560546875,
+      "rewards//std": 0.0270092636346817,
+      "step": 1543
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3088,
+      "grad_norm": 12.36461067199707,
+      "kl": 6.357189066708088,
+      "learning_rate": 7.918350271476063e-07,
+      "loss": 0.2543,
+      "num_tokens": 13345795.0,
+      "reward": 0.77752685546875,
+      "reward_std": 0.010957293212413788,
+      "rewards//mean": 0.77752685546875,
+      "rewards//std": 0.027272172272205353,
+      "step": 1544
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.309,
+      "grad_norm": 2.1590182781219482,
+      "kl": 2.6033093258738518,
+      "learning_rate": 7.915772972039659e-07,
+      "loss": 0.1041,
+      "num_tokens": 13354387.0,
+      "reward": 0.754150390625,
+      "reward_std": 0.006933914497494698,
+      "rewards//mean": 0.754150390625,
+      "rewards//std": 0.025428753346204758,
+      "step": 1545
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3092,
+      "grad_norm": 2.306547164916992,
+      "kl": 3.196092039346695,
+      "learning_rate": 7.913194498130251e-07,
+      "loss": 0.1278,
+      "num_tokens": 13363075.0,
+      "reward": 0.7781982421875,
+      "reward_std": 0.007881474681198597,
+      "rewards//mean": 0.7781982421875,
+      "rewards//std": 0.030060676857829094,
+      "step": 1546
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3094,
+      "grad_norm": 6.8349289894104,
+      "kl": 1.675778154283762,
+      "learning_rate": 7.910614850786447e-07,
+      "loss": 0.067,
+      "num_tokens": 13371691.0,
+      "reward": 0.7474365234375,
+      "reward_std": 0.0034942326601594687,
+      "rewards//mean": 0.7474365234375,
+      "rewards//std": 0.018390795215964317,
+      "step": 1547
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3096,
+      "grad_norm": 1.6075439453125,
+      "kl": 1.2292317170649767,
+      "learning_rate": 7.90803403104733e-07,
+      "loss": 0.0492,
+      "num_tokens": 13380339.0,
+      "reward": 0.76837158203125,
+      "reward_std": 0.0036253032740205526,
+      "rewards//mean": 0.76837158203125,
+      "rewards//std": 0.022731170058250427,
+      "step": 1548
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3098,
+      "grad_norm": 2.0693655014038086,
+      "kl": 2.8236304074525833,
+      "learning_rate": 7.905452039952451e-07,
+      "loss": 0.1129,
+      "num_tokens": 13388947.0,
+      "reward": 0.79052734375,
+      "reward_std": 0.008789876475930214,
+      "rewards//mean": 0.79052734375,
+      "rewards//std": 0.028977885842323303,
+      "step": 1549
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.31,
+      "grad_norm": 1.548401951789856,
+      "kl": 2.3623798824846745,
+      "learning_rate": 7.90286887854184e-07,
+      "loss": 0.0945,
+      "num_tokens": 13397571.0,
+      "reward": 0.75714111328125,
+      "reward_std": 0.0065483031794428825,
+      "rewards//mean": 0.75714111328125,
+      "rewards//std": 0.025128664448857307,
+      "step": 1550
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3102,
+      "grad_norm": 3.1247942447662354,
+      "kl": 2.1018613893538713,
+      "learning_rate": 7.900284547855991e-07,
+      "loss": 0.0841,
+      "num_tokens": 13406251.0,
+      "reward": 0.72906494140625,
+      "reward_std": 0.002748192986473441,
+      "rewards//mean": 0.72906494140625,
+      "rewards//std": 0.03136887773871422,
+      "step": 1551
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.546875,
+      "epoch": 0.3104,
+      "grad_norm": 3.9941465854644775,
+      "kl": 4.231639681383967,
+      "learning_rate": 7.897699048935873e-07,
+      "loss": 0.1661,
+      "num_tokens": 13414870.0,
+      "reward": 0.76788330078125,
+      "reward_std": 0.014554089866578579,
+      "rewards//mean": 0.76788330078125,
+      "rewards//std": 0.03652341291308403,
+      "step": 1552
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3106,
+      "grad_norm": 9.030753135681152,
+      "kl": 3.4044176265597343,
+      "learning_rate": 7.895112382822924e-07,
+      "loss": 0.1362,
+      "num_tokens": 13423446.0,
+      "reward": 0.7880859375,
+      "reward_std": 0.016512135043740273,
+      "rewards//mean": 0.7880859375,
+      "rewards//std": 0.03450608626008034,
+      "step": 1553
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3108,
+      "grad_norm": 1.22676682472229,
+      "kl": 1.1323249340057373,
+      "learning_rate": 7.892524550559055e-07,
+      "loss": 0.0453,
+      "num_tokens": 13432078.0,
+      "reward": 0.7481689453125,
+      "reward_std": 0.002001028275117278,
+      "rewards//mean": 0.7481689453125,
+      "rewards//std": 0.017538035288453102,
+      "step": 1554
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.311,
+      "grad_norm": 0.7332450747489929,
+      "kl": 1.2387255914509296,
+      "learning_rate": 7.889935553186641e-07,
+      "loss": 0.0495,
+      "num_tokens": 13440750.0,
+      "reward": 0.7340087890625,
+      "reward_std": 0.0024168689269572496,
+      "rewards//mean": 0.7340087890625,
+      "rewards//std": 0.039553701877593994,
+      "step": 1555
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3112,
+      "grad_norm": 4.305971145629883,
+      "kl": 5.2117039784789085,
+      "learning_rate": 7.887345391748532e-07,
+      "loss": 0.2085,
+      "num_tokens": 13449462.0,
+      "reward": 0.7496337890625,
+      "reward_std": 0.012276681140065193,
+      "rewards//mean": 0.7496337890625,
+      "rewards//std": 0.0312478207051754,
+      "step": 1556
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3114,
+      "grad_norm": 0.24466800689697266,
+      "kl": 0.5341769643127918,
+      "learning_rate": 7.884754067288046e-07,
+      "loss": 0.0214,
+      "num_tokens": 13457958.0,
+      "reward": 0.7513427734375,
+      "reward_std": 0.0004253430524840951,
+      "rewards//mean": 0.7513427734375,
+      "rewards//std": 0.019626734778285027,
+      "step": 1557
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3116,
+      "grad_norm": 3.6603965759277344,
+      "kl": 2.2707056906074286,
+      "learning_rate": 7.882161580848966e-07,
+      "loss": 0.0908,
+      "num_tokens": 13466590.0,
+      "reward": 0.77972412109375,
+      "reward_std": 0.003847534768283367,
+      "rewards//mean": 0.77972412109375,
+      "rewards//std": 0.029549933969974518,
+      "step": 1558
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3118,
+      "grad_norm": 10.382216453552246,
+      "kl": 5.203917965292931,
+      "learning_rate": 7.879567933475546e-07,
+      "loss": 0.2082,
+      "num_tokens": 13475222.0,
+      "reward": 0.7879638671875,
+      "reward_std": 0.0056140488013625145,
+      "rewards//mean": 0.7879638671875,
+      "rewards//std": 0.02510855160653591,
+      "step": 1559
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.312,
+      "grad_norm": 6.544485092163086,
+      "kl": 1.3014779221266508,
+      "learning_rate": 7.876973126212506e-07,
+      "loss": 0.0521,
+      "num_tokens": 13483806.0,
+      "reward": 0.77545166015625,
+      "reward_std": 0.005389615427702665,
+      "rewards//mean": 0.77545166015625,
+      "rewards//std": 0.02592974714934826,
+      "step": 1560
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3122,
+      "grad_norm": 2.880796194076538,
+      "kl": 3.00633068010211,
+      "learning_rate": 7.874377160105036e-07,
+      "loss": 0.1203,
+      "num_tokens": 13492502.0,
+      "reward": 0.73931884765625,
+      "reward_std": 0.007274072151631117,
+      "rewards//mean": 0.73931884765625,
+      "rewards//std": 0.0314142070710659,
+      "step": 1561
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3124,
+      "grad_norm": 10.305108070373535,
+      "kl": 3.9166896883398294,
+      "learning_rate": 7.871780036198788e-07,
+      "loss": 0.1567,
+      "num_tokens": 13501158.0,
+      "reward": 0.7200927734375,
+      "reward_std": 0.005273417569696903,
+      "rewards//mean": 0.7200927734375,
+      "rewards//std": 0.0368233397603035,
+      "step": 1562
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3126,
+      "grad_norm": 2.5040998458862305,
+      "kl": 4.020249454304576,
+      "learning_rate": 7.869181755539887e-07,
+      "loss": 0.1608,
+      "num_tokens": 13509718.0,
+      "reward": 0.78753662109375,
+      "reward_std": 0.014026065357029438,
+      "rewards//mean": 0.78753662109375,
+      "rewards//std": 0.0272560715675354,
+      "step": 1563
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3128,
+      "grad_norm": 4.298035621643066,
+      "kl": 3.5542656742036343,
+      "learning_rate": 7.866582319174917e-07,
+      "loss": 0.1422,
+      "num_tokens": 13518430.0,
+      "reward": 0.7562255859375,
+      "reward_std": 0.006951578892767429,
+      "rewards//mean": 0.7562255859375,
+      "rewards//std": 0.031263317912817,
+      "step": 1564
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.313,
+      "grad_norm": 4.246382236480713,
+      "kl": 2.2568327765911818,
+      "learning_rate": 7.863981728150931e-07,
+      "loss": 0.0903,
+      "num_tokens": 13527118.0,
+      "reward": 0.77130126953125,
+      "reward_std": 0.007427331525832415,
+      "rewards//mean": 0.77130126953125,
+      "rewards//std": 0.02944575995206833,
+      "step": 1565
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.046875,
+      "epoch": 0.3132,
+      "grad_norm": 12.954119682312012,
+      "kl": 2.3872750215232372,
+      "learning_rate": 7.861379983515448e-07,
+      "loss": 0.0804,
+      "num_tokens": 13535865.0,
+      "reward": 0.75164794921875,
+      "reward_std": 0.007791175507009029,
+      "rewards//mean": 0.75164794921875,
+      "rewards//std": 0.024258168414235115,
+      "step": 1566
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3134,
+      "grad_norm": 7.969794273376465,
+      "kl": 3.3750481456518173,
+      "learning_rate": 7.858777086316451e-07,
+      "loss": 0.135,
+      "num_tokens": 13544649.0,
+      "reward": 0.760498046875,
+      "reward_std": 0.006085801403969526,
+      "rewards//mean": 0.760498046875,
+      "rewards//std": 0.023927126079797745,
+      "step": 1567
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3136,
+      "grad_norm": 13.998820304870605,
+      "kl": 3.3849109187722206,
+      "learning_rate": 7.856173037602382e-07,
+      "loss": 0.1354,
+      "num_tokens": 13553329.0,
+      "reward": 0.74395751953125,
+      "reward_std": 0.008572198450565338,
+      "rewards//mean": 0.74395751953125,
+      "rewards//std": 0.031086266040802002,
+      "step": 1568
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3138,
+      "grad_norm": 6.771536350250244,
+      "kl": 2.87066737562418,
+      "learning_rate": 7.853567838422159e-07,
+      "loss": 0.1148,
+      "num_tokens": 13561985.0,
+      "reward": 0.7691650390625,
+      "reward_std": 0.00547350337728858,
+      "rewards//mean": 0.7691650390625,
+      "rewards//std": 0.02769770473241806,
+      "step": 1569
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.314,
+      "grad_norm": 12.953600883483887,
+      "kl": 5.080168567597866,
+      "learning_rate": 7.850961489825149e-07,
+      "loss": 0.2032,
+      "num_tokens": 13570689.0,
+      "reward": 0.71453857421875,
+      "reward_std": 0.007240781560540199,
+      "rewards//mean": 0.71453857421875,
+      "rewards//std": 0.02928389050066471,
+      "step": 1570
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3142,
+      "grad_norm": 14.087126731872559,
+      "kl": 5.163237893953919,
+      "learning_rate": 7.848353992861194e-07,
+      "loss": 0.2065,
+      "num_tokens": 13579393.0,
+      "reward": 0.7786865234375,
+      "reward_std": 0.006209178827702999,
+      "rewards//mean": 0.7786865234375,
+      "rewards//std": 0.02369667775928974,
+      "step": 1571
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3144,
+      "grad_norm": 2.7185895442962646,
+      "kl": 1.6526428014039993,
+      "learning_rate": 7.84574534858059e-07,
+      "loss": 0.0661,
+      "num_tokens": 13587945.0,
+      "reward": 0.79345703125,
+      "reward_std": 0.005284490995109081,
+      "rewards//mean": 0.79345703125,
+      "rewards//std": 0.01581757515668869,
+      "step": 1572
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3146,
+      "grad_norm": 6.753876209259033,
+      "kl": 1.8430143650621176,
+      "learning_rate": 7.8431355580341e-07,
+      "loss": 0.0737,
+      "num_tokens": 13596641.0,
+      "reward": 0.7794189453125,
+      "reward_std": 0.005295715294778347,
+      "rewards//mean": 0.7794189453125,
+      "rewards//std": 0.022977977991104126,
+      "step": 1573
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3148,
+      "grad_norm": 7.430104732513428,
+      "kl": 3.0881308056414127,
+      "learning_rate": 7.840524622272948e-07,
+      "loss": 0.1235,
+      "num_tokens": 13605297.0,
+      "reward": 0.75640869140625,
+      "reward_std": 0.01122256275266409,
+      "rewards//mean": 0.75640869140625,
+      "rewards//std": 0.02808542177081108,
+      "step": 1574
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.315,
+      "grad_norm": 3.793759822845459,
+      "kl": 2.775339739397168,
+      "learning_rate": 7.837912542348817e-07,
+      "loss": 0.111,
+      "num_tokens": 13613921.0,
+      "reward": 0.802978515625,
+      "reward_std": 0.008220259100198746,
+      "rewards//mean": 0.802978515625,
+      "rewards//std": 0.029139375314116478,
+      "step": 1575
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3152,
+      "grad_norm": 7.971090316772461,
+      "kl": 3.0225213952362537,
+      "learning_rate": 7.835299319313853e-07,
+      "loss": 0.1209,
+      "num_tokens": 13622457.0,
+      "reward": 0.78424072265625,
+      "reward_std": 0.008467039093375206,
+      "rewards//mean": 0.78424072265625,
+      "rewards//std": 0.026709645986557007,
+      "step": 1576
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3154,
+      "grad_norm": 7.319133281707764,
+      "kl": 4.196964494884014,
+      "learning_rate": 7.832684954220663e-07,
+      "loss": 0.1679,
+      "num_tokens": 13631257.0,
+      "reward": 0.764892578125,
+      "reward_std": 0.008041007444262505,
+      "rewards//mean": 0.764892578125,
+      "rewards//std": 0.022454533725976944,
+      "step": 1577
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3156,
+      "grad_norm": 6.501193046569824,
+      "kl": 0.7581717055290937,
+      "learning_rate": 7.830069448122312e-07,
+      "loss": 0.0303,
+      "num_tokens": 13639849.0,
+      "reward": 0.78497314453125,
+      "reward_std": 0.002913340460509062,
+      "rewards//mean": 0.78497314453125,
+      "rewards//std": 0.01798073947429657,
+      "step": 1578
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3158,
+      "grad_norm": 11.395877838134766,
+      "kl": 4.177056362852454,
+      "learning_rate": 7.827452802072327e-07,
+      "loss": 0.1671,
+      "num_tokens": 13648569.0,
+      "reward": 0.74993896484375,
+      "reward_std": 0.008374565280973911,
+      "rewards//mean": 0.74993896484375,
+      "rewards//std": 0.027432115748524666,
+      "step": 1579
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.316,
+      "grad_norm": 6.650692462921143,
+      "kl": 3.6675681360065937,
+      "learning_rate": 7.82483501712469e-07,
+      "loss": 0.1467,
+      "num_tokens": 13657185.0,
+      "reward": 0.72760009765625,
+      "reward_std": 0.007356700487434864,
+      "rewards//mean": 0.72760009765625,
+      "rewards//std": 0.03624005988240242,
+      "step": 1580
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3162,
+      "grad_norm": 16.119176864624023,
+      "kl": 2.5368894282728434,
+      "learning_rate": 7.822216094333847e-07,
+      "loss": 0.1015,
+      "num_tokens": 13665769.0,
+      "reward": 0.7923583984375,
+      "reward_std": 0.008061882108449936,
+      "rewards//mean": 0.7923583984375,
+      "rewards//std": 0.024815035983920097,
+      "step": 1581
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3164,
+      "grad_norm": 8.031620025634766,
+      "kl": 4.719241995364428,
+      "learning_rate": 7.819596034754696e-07,
+      "loss": 0.1888,
+      "num_tokens": 13674369.0,
+      "reward": 0.76568603515625,
+      "reward_std": 0.012494444847106934,
+      "rewards//mean": 0.76568603515625,
+      "rewards//std": 0.028172602877020836,
+      "step": 1582
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3166,
+      "grad_norm": 12.50458812713623,
+      "kl": 5.18662153929472,
+      "learning_rate": 7.816974839442603e-07,
+      "loss": 0.2075,
+      "num_tokens": 13683049.0,
+      "reward": 0.75146484375,
+      "reward_std": 0.01645568758249283,
+      "rewards//mean": 0.75146484375,
+      "rewards//std": 0.03689746931195259,
+      "step": 1583
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3168,
+      "grad_norm": 4.855544090270996,
+      "kl": 3.400411695241928,
+      "learning_rate": 7.814352509453379e-07,
+      "loss": 0.136,
+      "num_tokens": 13691641.0,
+      "reward": 0.76513671875,
+      "reward_std": 0.009719028137624264,
+      "rewards//mean": 0.76513671875,
+      "rewards//std": 0.028301332145929337,
+      "step": 1584
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.317,
+      "grad_norm": 10.149500846862793,
+      "kl": 2.5843121875077486,
+      "learning_rate": 7.811729045843301e-07,
+      "loss": 0.1034,
+      "num_tokens": 13700241.0,
+      "reward": 0.7584228515625,
+      "reward_std": 0.006244299001991749,
+      "rewards//mean": 0.7584228515625,
+      "rewards//std": 0.029718313366174698,
+      "step": 1585
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3172,
+      "grad_norm": 7.332499980926514,
+      "kl": 2.5256293322890997,
+      "learning_rate": 7.8091044496691e-07,
+      "loss": 0.101,
+      "num_tokens": 13708873.0,
+      "reward": 0.74774169921875,
+      "reward_std": 0.008119644597172737,
+      "rewards//mean": 0.74774169921875,
+      "rewards//std": 0.03190051019191742,
+      "step": 1586
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3174,
+      "grad_norm": 8.883455276489258,
+      "kl": 2.6917338632047176,
+      "learning_rate": 7.806478721987963e-07,
+      "loss": 0.1077,
+      "num_tokens": 13717625.0,
+      "reward": 0.7562255859375,
+      "reward_std": 0.006581498775631189,
+      "rewards//mean": 0.7562255859375,
+      "rewards//std": 0.023691566661000252,
+      "step": 1587
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3176,
+      "grad_norm": 10.507652282714844,
+      "kl": 2.3051068540662527,
+      "learning_rate": 7.803851863857532e-07,
+      "loss": 0.0922,
+      "num_tokens": 13726369.0,
+      "reward": 0.75238037109375,
+      "reward_std": 0.006877091713249683,
+      "rewards//mean": 0.75238037109375,
+      "rewards//std": 0.02831944450736046,
+      "step": 1588
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3178,
+      "grad_norm": 6.7308173179626465,
+      "kl": 1.5977623760700226,
+      "learning_rate": 7.801223876335907e-07,
+      "loss": 0.0639,
+      "num_tokens": 13735057.0,
+      "reward": 0.758056640625,
+      "reward_std": 0.007009593769907951,
+      "rewards//mean": 0.758056640625,
+      "rewards//std": 0.023259857669472694,
+      "step": 1589
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.318,
+      "grad_norm": 15.043256759643555,
+      "kl": 4.218444623053074,
+      "learning_rate": 7.798594760481637e-07,
+      "loss": 0.1687,
+      "num_tokens": 13743641.0,
+      "reward": 0.7542724609375,
+      "reward_std": 0.010292530059814453,
+      "rewards//mean": 0.7542724609375,
+      "rewards//std": 0.02739877812564373,
+      "step": 1590
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3182,
+      "grad_norm": 25.150230407714844,
+      "kl": 6.923338778316975,
+      "learning_rate": 7.795964517353733e-07,
+      "loss": 0.2769,
+      "num_tokens": 13752361.0,
+      "reward": 0.74615478515625,
+      "reward_std": 0.009582172147929668,
+      "rewards//mean": 0.74615478515625,
+      "rewards//std": 0.024144954979419708,
+      "step": 1591
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3184,
+      "grad_norm": 35.85035705566406,
+      "kl": 5.761877480894327,
+      "learning_rate": 7.793333148011657e-07,
+      "loss": 0.2305,
+      "num_tokens": 13760945.0,
+      "reward": 0.74853515625,
+      "reward_std": 0.007426196709275246,
+      "rewards//mean": 0.74853515625,
+      "rewards//std": 0.030675049871206284,
+      "step": 1592
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3186,
+      "grad_norm": 7.998895645141602,
+      "kl": 4.327491544187069,
+      "learning_rate": 7.790700653515323e-07,
+      "loss": 0.1731,
+      "num_tokens": 13769489.0,
+      "reward": 0.71014404296875,
+      "reward_std": 0.013705659657716751,
+      "rewards//mean": 0.71014404296875,
+      "rewards//std": 0.037732694298028946,
+      "step": 1593
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3188,
+      "grad_norm": 18.91761016845703,
+      "kl": 4.072094418108463,
+      "learning_rate": 7.788067034925099e-07,
+      "loss": 0.1629,
+      "num_tokens": 13778065.0,
+      "reward": 0.7537841796875,
+      "reward_std": 0.01519739255309105,
+      "rewards//mean": 0.7537841796875,
+      "rewards//std": 0.028763698413968086,
+      "step": 1594
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.319,
+      "grad_norm": 14.40042781829834,
+      "kl": 4.263888690620661,
+      "learning_rate": 7.785432293301806e-07,
+      "loss": 0.1706,
+      "num_tokens": 13786681.0,
+      "reward": 0.728759765625,
+      "reward_std": 0.008491072803735733,
+      "rewards//mean": 0.728759765625,
+      "rewards//std": 0.0288217943161726,
+      "step": 1595
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3192,
+      "grad_norm": 5.8197808265686035,
+      "kl": 3.3678077179938555,
+      "learning_rate": 7.78279642970672e-07,
+      "loss": 0.1347,
+      "num_tokens": 13795241.0,
+      "reward": 0.7589111328125,
+      "reward_std": 0.011701389215886593,
+      "rewards//mean": 0.7589111328125,
+      "rewards//std": 0.03187517076730728,
+      "step": 1596
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3194,
+      "grad_norm": 24.04300308227539,
+      "kl": 2.8448747098445892,
+      "learning_rate": 7.780159445201562e-07,
+      "loss": 0.1138,
+      "num_tokens": 13803929.0,
+      "reward": 0.80096435546875,
+      "reward_std": 0.01415582187473774,
+      "rewards//mean": 0.80096435546875,
+      "rewards//std": 0.026842502877116203,
+      "step": 1597
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3196,
+      "grad_norm": 6.142871856689453,
+      "kl": 2.120186196640134,
+      "learning_rate": 7.777521340848514e-07,
+      "loss": 0.0848,
+      "num_tokens": 13812577.0,
+      "reward": 0.71734619140625,
+      "reward_std": 0.008194508031010628,
+      "rewards//mean": 0.71734619140625,
+      "rewards//std": 0.032692164182662964,
+      "step": 1598
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3198,
+      "grad_norm": 3.9844765663146973,
+      "kl": 1.9246549867093563,
+      "learning_rate": 7.774882117710202e-07,
+      "loss": 0.077,
+      "num_tokens": 13821161.0,
+      "reward": 0.749267578125,
+      "reward_std": 0.005446576979011297,
+      "rewards//mean": 0.749267578125,
+      "rewards//std": 0.024928545579314232,
+      "step": 1599
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.32,
+      "grad_norm": 7.272011756896973,
+      "kl": 2.294439125806093,
+      "learning_rate": 7.772241776849704e-07,
+      "loss": 0.0918,
+      "num_tokens": 13829705.0,
+      "reward": 0.7069091796875,
+      "reward_std": 0.007744522299617529,
+      "rewards//mean": 0.7069091796875,
+      "rewards//std": 0.02167622372508049,
+      "step": 1600
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3202,
+      "grad_norm": 7.847100257873535,
+      "kl": 3.460626281797886,
+      "learning_rate": 7.769600319330552e-07,
+      "loss": 0.1384,
+      "num_tokens": 13838377.0,
+      "reward": 0.76690673828125,
+      "reward_std": 0.012261524796485901,
+      "rewards//mean": 0.76690673828125,
+      "rewards//std": 0.03251154348254204,
+      "step": 1601
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3204,
+      "grad_norm": 13.213078498840332,
+      "kl": 3.530375551432371,
+      "learning_rate": 7.76695774621672e-07,
+      "loss": 0.1412,
+      "num_tokens": 13847049.0,
+      "reward": 0.75537109375,
+      "reward_std": 0.009636061266064644,
+      "rewards//mean": 0.75537109375,
+      "rewards//std": 0.03106733039021492,
+      "step": 1602
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3206,
+      "grad_norm": 7.9046478271484375,
+      "kl": 2.3466283716261387,
+      "learning_rate": 7.764314058572639e-07,
+      "loss": 0.0939,
+      "num_tokens": 13855689.0,
+      "reward": 0.75860595703125,
+      "reward_std": 0.00639187078922987,
+      "rewards//mean": 0.75860595703125,
+      "rewards//std": 0.016879508271813393,
+      "step": 1603
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3208,
+      "grad_norm": 11.421862602233887,
+      "kl": 3.2277444936335087,
+      "learning_rate": 7.761669257463187e-07,
+      "loss": 0.1291,
+      "num_tokens": 13864481.0,
+      "reward": 0.7777099609375,
+      "reward_std": 0.012739087454974651,
+      "rewards//mean": 0.7777099609375,
+      "rewards//std": 0.01985372230410576,
+      "step": 1604
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.321,
+      "grad_norm": 22.2473201751709,
+      "kl": 3.5649806205183268,
+      "learning_rate": 7.759023343953688e-07,
+      "loss": 0.1426,
+      "num_tokens": 13873025.0,
+      "reward": 0.75970458984375,
+      "reward_std": 0.013574929907917976,
+      "rewards//mean": 0.75970458984375,
+      "rewards//std": 0.030040716752409935,
+      "step": 1605
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3212,
+      "grad_norm": 25.993104934692383,
+      "kl": 7.29346152022481,
+      "learning_rate": 7.756376319109916e-07,
+      "loss": 0.2917,
+      "num_tokens": 13881697.0,
+      "reward": 0.73590087890625,
+      "reward_std": 0.011688966304063797,
+      "rewards//mean": 0.73590087890625,
+      "rewards//std": 0.026389211416244507,
+      "step": 1606
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3214,
+      "grad_norm": 18.03403663635254,
+      "kl": 5.4808325953781605,
+      "learning_rate": 7.753728183998092e-07,
+      "loss": 0.2192,
+      "num_tokens": 13890337.0,
+      "reward": 0.75201416015625,
+      "reward_std": 0.013721934519708157,
+      "rewards//mean": 0.75201416015625,
+      "rewards//std": 0.030804993584752083,
+      "step": 1607
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3216,
+      "grad_norm": 31.654752731323242,
+      "kl": 6.4295050743967295,
+      "learning_rate": 7.751078939684885e-07,
+      "loss": 0.2572,
+      "num_tokens": 13898905.0,
+      "reward": 0.75146484375,
+      "reward_std": 0.011243881657719612,
+      "rewards//mean": 0.75146484375,
+      "rewards//std": 0.02818985842168331,
+      "step": 1608
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3218,
+      "grad_norm": 59.448848724365234,
+      "kl": 13.091281961649656,
+      "learning_rate": 7.748428587237411e-07,
+      "loss": 0.5237,
+      "num_tokens": 13907505.0,
+      "reward": 0.73382568359375,
+      "reward_std": 0.01739353872835636,
+      "rewards//mean": 0.73382568359375,
+      "rewards//std": 0.030120227485895157,
+      "step": 1609
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.322,
+      "grad_norm": 41.87702178955078,
+      "kl": 7.90415133535862,
+      "learning_rate": 7.74577712772323e-07,
+      "loss": 0.3162,
+      "num_tokens": 13916113.0,
+      "reward": 0.7431640625,
+      "reward_std": 0.01467541791498661,
+      "rewards//mean": 0.7431640625,
+      "rewards//std": 0.0174969881772995,
+      "step": 1610
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3222,
+      "grad_norm": 18.246580123901367,
+      "kl": 5.010819291695952,
+      "learning_rate": 7.743124562210351e-07,
+      "loss": 0.2004,
+      "num_tokens": 13924729.0,
+      "reward": 0.713134765625,
+      "reward_std": 0.013421990908682346,
+      "rewards//mean": 0.713134765625,
+      "rewards//std": 0.026216628029942513,
+      "step": 1611
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3224,
+      "grad_norm": 11.278694152832031,
+      "kl": 3.67126352712512,
+      "learning_rate": 7.740470891767224e-07,
+      "loss": 0.1469,
+      "num_tokens": 13933313.0,
+      "reward": 0.7525634765625,
+      "reward_std": 0.006975987926125526,
+      "rewards//mean": 0.7525634765625,
+      "rewards//std": 0.02472703717648983,
+      "step": 1612
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3226,
+      "grad_norm": 21.367422103881836,
+      "kl": 4.67998156696558,
+      "learning_rate": 7.737816117462751e-07,
+      "loss": 0.1872,
+      "num_tokens": 13941849.0,
+      "reward": 0.74945068359375,
+      "reward_std": 0.007777191698551178,
+      "rewards//mean": 0.74945068359375,
+      "rewards//std": 0.030590981245040894,
+      "step": 1613
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3228,
+      "grad_norm": 14.104653358459473,
+      "kl": 4.814919566735625,
+      "learning_rate": 7.735160240366274e-07,
+      "loss": 0.1926,
+      "num_tokens": 13950593.0,
+      "reward": 0.7359619140625,
+      "reward_std": 0.007620183285325766,
+      "rewards//mean": 0.7359619140625,
+      "rewards//std": 0.027802443131804466,
+      "step": 1614
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.323,
+      "grad_norm": 5.918192386627197,
+      "kl": 3.786400757730007,
+      "learning_rate": 7.732503261547578e-07,
+      "loss": 0.1515,
+      "num_tokens": 13959273.0,
+      "reward": 0.77899169921875,
+      "reward_std": 0.009284375235438347,
+      "rewards//mean": 0.77899169921875,
+      "rewards//std": 0.022859998047351837,
+      "step": 1615
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3232,
+      "grad_norm": 29.60202407836914,
+      "kl": 2.178092934191227,
+      "learning_rate": 7.729845182076895e-07,
+      "loss": 0.0871,
+      "num_tokens": 13967897.0,
+      "reward": 0.79571533203125,
+      "reward_std": 0.017498420551419258,
+      "rewards//mean": 0.79571533203125,
+      "rewards//std": 0.02256004326045513,
+      "step": 1616
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3234,
+      "grad_norm": 15.12800121307373,
+      "kl": 2.791096108034253,
+      "learning_rate": 7.7271860030249e-07,
+      "loss": 0.1116,
+      "num_tokens": 13976521.0,
+      "reward": 0.7176513671875,
+      "reward_std": 0.015062645077705383,
+      "rewards//mean": 0.7176513671875,
+      "rewards//std": 0.038934990763664246,
+      "step": 1617
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3236,
+      "grad_norm": 8.760200500488281,
+      "kl": 3.269177110865712,
+      "learning_rate": 7.72452572546271e-07,
+      "loss": 0.1308,
+      "num_tokens": 13985233.0,
+      "reward": 0.7557373046875,
+      "reward_std": 0.009893681854009628,
+      "rewards//mean": 0.7557373046875,
+      "rewards//std": 0.03662383556365967,
+      "step": 1618
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3238,
+      "grad_norm": 7.397759914398193,
+      "kl": 3.589187180623412,
+      "learning_rate": 7.721864350461882e-07,
+      "loss": 0.1436,
+      "num_tokens": 13993897.0,
+      "reward": 0.7772216796875,
+      "reward_std": 0.018703140318393707,
+      "rewards//mean": 0.7772216796875,
+      "rewards//std": 0.03549542278051376,
+      "step": 1619
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.324,
+      "grad_norm": 8.775015830993652,
+      "kl": 1.8012400344014168,
+      "learning_rate": 7.71920187909442e-07,
+      "loss": 0.072,
+      "num_tokens": 14002497.0,
+      "reward": 0.75531005859375,
+      "reward_std": 0.006133618764579296,
+      "rewards//mean": 0.75531005859375,
+      "rewards//std": 0.029358750209212303,
+      "step": 1620
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3242,
+      "grad_norm": 11.978819847106934,
+      "kl": 5.873415801674128,
+      "learning_rate": 7.716538312432765e-07,
+      "loss": 0.2349,
+      "num_tokens": 14011017.0,
+      "reward": 0.73675537109375,
+      "reward_std": 0.019236888736486435,
+      "rewards//mean": 0.73675537109375,
+      "rewards//std": 0.030715428292751312,
+      "step": 1621
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3244,
+      "grad_norm": 6.883742332458496,
+      "kl": 3.0566181652247906,
+      "learning_rate": 7.713873651549804e-07,
+      "loss": 0.1223,
+      "num_tokens": 14019617.0,
+      "reward": 0.7467041015625,
+      "reward_std": 0.009721335954964161,
+      "rewards//mean": 0.7467041015625,
+      "rewards//std": 0.027234749868512154,
+      "step": 1622
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3246,
+      "grad_norm": 8.357880592346191,
+      "kl": 1.9255514815449715,
+      "learning_rate": 7.71120789751886e-07,
+      "loss": 0.077,
+      "num_tokens": 14028185.0,
+      "reward": 0.75762939453125,
+      "reward_std": 0.01105282362550497,
+      "rewards//mean": 0.75762939453125,
+      "rewards//std": 0.027510913088917732,
+      "step": 1623
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3248,
+      "grad_norm": 7.992527484893799,
+      "kl": 2.4378102142363787,
+      "learning_rate": 7.7085410514137e-07,
+      "loss": 0.0975,
+      "num_tokens": 14036937.0,
+      "reward": 0.750244140625,
+      "reward_std": 0.009366312995553017,
+      "rewards//mean": 0.750244140625,
+      "rewards//std": 0.03406723216176033,
+      "step": 1624
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.325,
+      "grad_norm": 12.08099365234375,
+      "kl": 2.1106253564357758,
+      "learning_rate": 7.705873114308527e-07,
+      "loss": 0.0844,
+      "num_tokens": 14045609.0,
+      "reward": 0.761962890625,
+      "reward_std": 0.00817366223782301,
+      "rewards//mean": 0.761962890625,
+      "rewards//std": 0.01795978471636772,
+      "step": 1625
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3252,
+      "grad_norm": 9.843433380126953,
+      "kl": 2.6523684319108725,
+      "learning_rate": 7.703204087277988e-07,
+      "loss": 0.1061,
+      "num_tokens": 14054305.0,
+      "reward": 0.75311279296875,
+      "reward_std": 0.011103453114628792,
+      "rewards//mean": 0.75311279296875,
+      "rewards//std": 0.022110015153884888,
+      "step": 1626
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3254,
+      "grad_norm": 10.099394798278809,
+      "kl": 1.7653774097561836,
+      "learning_rate": 7.700533971397165e-07,
+      "loss": 0.0706,
+      "num_tokens": 14062945.0,
+      "reward": 0.73114013671875,
+      "reward_std": 0.006639300845563412,
+      "rewards//mean": 0.73114013671875,
+      "rewards//std": 0.030762212350964546,
+      "step": 1627
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3256,
+      "grad_norm": 6.093928813934326,
+      "kl": 1.878224153071642,
+      "learning_rate": 7.697862767741583e-07,
+      "loss": 0.0751,
+      "num_tokens": 14071561.0,
+      "reward": 0.7745361328125,
+      "reward_std": 0.007871415466070175,
+      "rewards//mean": 0.7745361328125,
+      "rewards//std": 0.026212874799966812,
+      "step": 1628
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3258,
+      "grad_norm": 5.065017223358154,
+      "kl": 1.2661594208329916,
+      "learning_rate": 7.695190477387199e-07,
+      "loss": 0.0506,
+      "num_tokens": 14080193.0,
+      "reward": 0.7513427734375,
+      "reward_std": 0.004636221565306187,
+      "rewards//mean": 0.7513427734375,
+      "rewards//std": 0.020484179258346558,
+      "step": 1629
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.326,
+      "grad_norm": 27.482603073120117,
+      "kl": 3.3264848925173283,
+      "learning_rate": 7.692517101410414e-07,
+      "loss": 0.1331,
+      "num_tokens": 14088921.0,
+      "reward": 0.77496337890625,
+      "reward_std": 0.015468500554561615,
+      "rewards//mean": 0.77496337890625,
+      "rewards//std": 0.027220504358410835,
+      "step": 1630
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3262,
+      "grad_norm": 10.311243057250977,
+      "kl": 3.5869624000042677,
+      "learning_rate": 7.689842640888063e-07,
+      "loss": 0.1435,
+      "num_tokens": 14097505.0,
+      "reward": 0.7471923828125,
+      "reward_std": 0.009189065545797348,
+      "rewards//mean": 0.7471923828125,
+      "rewards//std": 0.02355572022497654,
+      "step": 1631
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3264,
+      "grad_norm": 9.306674003601074,
+      "kl": 3.5584700033068657,
+      "learning_rate": 7.687167096897418e-07,
+      "loss": 0.1423,
+      "num_tokens": 14106249.0,
+      "reward": 0.7293701171875,
+      "reward_std": 0.01151098869740963,
+      "rewards//mean": 0.7293701171875,
+      "rewards//std": 0.02178211510181427,
+      "step": 1632
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3266,
+      "grad_norm": 13.561746597290039,
+      "kl": 2.1063948441296816,
+      "learning_rate": 7.684490470516185e-07,
+      "loss": 0.0843,
+      "num_tokens": 14114817.0,
+      "reward": 0.76678466796875,
+      "reward_std": 0.004899102263152599,
+      "rewards//mean": 0.76678466796875,
+      "rewards//std": 0.021739942952990532,
+      "step": 1633
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3268,
+      "grad_norm": 7.863563537597656,
+      "kl": 4.624258419498801,
+      "learning_rate": 7.681812762822515e-07,
+      "loss": 0.185,
+      "num_tokens": 14123481.0,
+      "reward": 0.75518798828125,
+      "reward_std": 0.012642089277505875,
+      "rewards//mean": 0.75518798828125,
+      "rewards//std": 0.03485565632581711,
+      "step": 1634
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.327,
+      "grad_norm": 22.702239990234375,
+      "kl": 2.840575873851776,
+      "learning_rate": 7.679133974894982e-07,
+      "loss": 0.1136,
+      "num_tokens": 14132065.0,
+      "reward": 0.75201416015625,
+      "reward_std": 0.00959020759910345,
+      "rewards//mean": 0.75201416015625,
+      "rewards//std": 0.028237005695700645,
+      "step": 1635
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3272,
+      "grad_norm": 5.975203990936279,
+      "kl": 2.8444022815674543,
+      "learning_rate": 7.676454107812607e-07,
+      "loss": 0.1138,
+      "num_tokens": 14140681.0,
+      "reward": 0.78369140625,
+      "reward_std": 0.009267503395676613,
+      "rewards//mean": 0.78369140625,
+      "rewards//std": 0.02929377555847168,
+      "step": 1636
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3274,
+      "grad_norm": 14.63658332824707,
+      "kl": 7.024058857932687,
+      "learning_rate": 7.673773162654836e-07,
+      "loss": 0.281,
+      "num_tokens": 14149393.0,
+      "reward": 0.749755859375,
+      "reward_std": 0.011115207336843014,
+      "rewards//mean": 0.749755859375,
+      "rewards//std": 0.029926680028438568,
+      "step": 1637
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3276,
+      "grad_norm": 12.250699043273926,
+      "kl": 3.8545846436172724,
+      "learning_rate": 7.671091140501555e-07,
+      "loss": 0.1542,
+      "num_tokens": 14158001.0,
+      "reward": 0.75225830078125,
+      "reward_std": 0.011953351087868214,
+      "rewards//mean": 0.75225830078125,
+      "rewards//std": 0.021062320098280907,
+      "step": 1638
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.265625,
+      "epoch": 0.3278,
+      "grad_norm": 14.134915351867676,
+      "kl": 5.8936877734959126,
+      "learning_rate": 7.668408042433081e-07,
+      "loss": 0.224,
+      "num_tokens": 14166562.0,
+      "reward": 0.7552490234375,
+      "reward_std": 0.009656651876866817,
+      "rewards//mean": 0.7552490234375,
+      "rewards//std": 0.03004254214465618,
+      "step": 1639
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.328,
+      "grad_norm": 10.148066520690918,
+      "kl": 6.543967846781015,
+      "learning_rate": 7.665723869530169e-07,
+      "loss": 0.2618,
+      "num_tokens": 14175130.0,
+      "reward": 0.76983642578125,
+      "reward_std": 0.014795171096920967,
+      "rewards//mean": 0.76983642578125,
+      "rewards//std": 0.026994338259100914,
+      "step": 1640
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3282,
+      "grad_norm": 26.754764556884766,
+      "kl": 8.376775816082954,
+      "learning_rate": 7.663038622873999e-07,
+      "loss": 0.3351,
+      "num_tokens": 14183738.0,
+      "reward": 0.740234375,
+      "reward_std": 0.014836485497653484,
+      "rewards//mean": 0.740234375,
+      "rewards//std": 0.03266720846295357,
+      "step": 1641
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.546875,
+      "epoch": 0.3284,
+      "grad_norm": 11.834562301635742,
+      "kl": 4.77639720775187,
+      "learning_rate": 7.660352303546192e-07,
+      "loss": 0.1959,
+      "num_tokens": 14192349.0,
+      "reward": 0.72686767578125,
+      "reward_std": 0.010114647448062897,
+      "rewards//mean": 0.72686767578125,
+      "rewards//std": 0.03528556600213051,
+      "step": 1642
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3286,
+      "grad_norm": 27.761306762695312,
+      "kl": 9.532374881207943,
+      "learning_rate": 7.657664912628794e-07,
+      "loss": 0.3813,
+      "num_tokens": 14201133.0,
+      "reward": 0.7391357421875,
+      "reward_std": 0.01447632908821106,
+      "rewards//mean": 0.7391357421875,
+      "rewards//std": 0.03351961448788643,
+      "step": 1643
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3288,
+      "grad_norm": 12.942228317260742,
+      "kl": 5.6833315175026655,
+      "learning_rate": 7.654976451204287e-07,
+      "loss": 0.2273,
+      "num_tokens": 14209693.0,
+      "reward": 0.72845458984375,
+      "reward_std": 0.011466937139630318,
+      "rewards//mean": 0.72845458984375,
+      "rewards//std": 0.030040716752409935,
+      "step": 1644
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.329,
+      "grad_norm": 8.533147811889648,
+      "kl": 3.2423823177814484,
+      "learning_rate": 7.652286920355583e-07,
+      "loss": 0.1297,
+      "num_tokens": 14218405.0,
+      "reward": 0.7584228515625,
+      "reward_std": 0.010816293768584728,
+      "rewards//mean": 0.7584228515625,
+      "rewards//std": 0.033539481461048126,
+      "step": 1645
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3292,
+      "grad_norm": 19.20831298828125,
+      "kl": 5.504130233079195,
+      "learning_rate": 7.649596321166024e-07,
+      "loss": 0.2202,
+      "num_tokens": 14227165.0,
+      "reward": 0.76483154296875,
+      "reward_std": 0.011215681210160255,
+      "rewards//mean": 0.76483154296875,
+      "rewards//std": 0.025996796786785126,
+      "step": 1646
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3294,
+      "grad_norm": 6.512764930725098,
+      "kl": 4.2323793433606625,
+      "learning_rate": 7.646904654719385e-07,
+      "loss": 0.1693,
+      "num_tokens": 14235821.0,
+      "reward": 0.7633056640625,
+      "reward_std": 0.01945539563894272,
+      "rewards//mean": 0.7633056640625,
+      "rewards//std": 0.026854129508137703,
+      "step": 1647
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.984375,
+      "epoch": 0.3296,
+      "grad_norm": 7.150025367736816,
+      "kl": 3.510749952867627,
+      "learning_rate": 7.644211922099867e-07,
+      "loss": 0.1404,
+      "num_tokens": 14244436.0,
+      "reward": 0.74188232421875,
+      "reward_std": 0.0065304902382195,
+      "rewards//mean": 0.74188232421875,
+      "rewards//std": 0.022737829014658928,
+      "step": 1648
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3298,
+      "grad_norm": 18.398927688598633,
+      "kl": 4.953672690317035,
+      "learning_rate": 7.641518124392103e-07,
+      "loss": 0.1981,
+      "num_tokens": 14253076.0,
+      "reward": 0.7508544921875,
+      "reward_std": 0.005613813176751137,
+      "rewards//mean": 0.7508544921875,
+      "rewards//std": 0.018939048051834106,
+      "step": 1649
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.33,
+      "grad_norm": 5.904971122741699,
+      "kl": 2.5764774549752474,
+      "learning_rate": 7.638823262681154e-07,
+      "loss": 0.1031,
+      "num_tokens": 14261660.0,
+      "reward": 0.77178955078125,
+      "reward_std": 0.010225379839539528,
+      "rewards//mean": 0.77178955078125,
+      "rewards//std": 0.031217467039823532,
+      "step": 1650
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3302,
+      "grad_norm": 11.138781547546387,
+      "kl": 3.0971457418054342,
+      "learning_rate": 7.636127338052511e-07,
+      "loss": 0.1239,
+      "num_tokens": 14270276.0,
+      "reward": 0.7667236328125,
+      "reward_std": 0.012113604694604874,
+      "rewards//mean": 0.7667236328125,
+      "rewards//std": 0.027396569028496742,
+      "step": 1651
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3304,
+      "grad_norm": 10.846475601196289,
+      "kl": 2.1628370974212885,
+      "learning_rate": 7.633430351592093e-07,
+      "loss": 0.0865,
+      "num_tokens": 14278900.0,
+      "reward": 0.75048828125,
+      "reward_std": 0.004749837331473827,
+      "rewards//mean": 0.75048828125,
+      "rewards//std": 0.028641607612371445,
+      "step": 1652
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3306,
+      "grad_norm": 15.916720390319824,
+      "kl": 1.1707215327769518,
+      "learning_rate": 7.630732304386243e-07,
+      "loss": 0.0468,
+      "num_tokens": 14287516.0,
+      "reward": 0.77703857421875,
+      "reward_std": 0.004662022925913334,
+      "rewards//mean": 0.77703857421875,
+      "rewards//std": 0.027039162814617157,
+      "step": 1653
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3308,
+      "grad_norm": 22.2278995513916,
+      "kl": 2.5529266484081745,
+      "learning_rate": 7.628033197521735e-07,
+      "loss": 0.1021,
+      "num_tokens": 14296180.0,
+      "reward": 0.74749755859375,
+      "reward_std": 0.008782346732914448,
+      "rewards//mean": 0.74749755859375,
+      "rewards//std": 0.019995320588350296,
+      "step": 1654
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.331,
+      "grad_norm": 15.795402526855469,
+      "kl": 1.6755111087113619,
+      "learning_rate": 7.625333032085769e-07,
+      "loss": 0.067,
+      "num_tokens": 14304812.0,
+      "reward": 0.74957275390625,
+      "reward_std": 0.008638689294457436,
+      "rewards//mean": 0.74957275390625,
+      "rewards//std": 0.022607633844017982,
+      "step": 1655
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3312,
+      "grad_norm": 6.203474521636963,
+      "kl": 2.146116741001606,
+      "learning_rate": 7.622631809165972e-07,
+      "loss": 0.0858,
+      "num_tokens": 14313468.0,
+      "reward": 0.77423095703125,
+      "reward_std": 0.010866011492908001,
+      "rewards//mean": 0.77423095703125,
+      "rewards//std": 0.026722678914666176,
+      "step": 1656
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3314,
+      "grad_norm": 6.657282829284668,
+      "kl": 2.3803008049726486,
+      "learning_rate": 7.619929529850396e-07,
+      "loss": 0.0952,
+      "num_tokens": 14322052.0,
+      "reward": 0.7696533203125,
+      "reward_std": 0.006360728293657303,
+      "rewards//mean": 0.7696533203125,
+      "rewards//std": 0.025901490822434425,
+      "step": 1657
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3316,
+      "grad_norm": 8.862725257873535,
+      "kl": 2.4458378311246634,
+      "learning_rate": 7.617226195227517e-07,
+      "loss": 0.0978,
+      "num_tokens": 14330684.0,
+      "reward": 0.778076171875,
+      "reward_std": 0.006107522174715996,
+      "rewards//mean": 0.778076171875,
+      "rewards//std": 0.018905848264694214,
+      "step": 1658
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3318,
+      "grad_norm": 8.068026542663574,
+      "kl": 3.9540002550929785,
+      "learning_rate": 7.614521806386243e-07,
+      "loss": 0.1582,
+      "num_tokens": 14339292.0,
+      "reward": 0.74945068359375,
+      "reward_std": 0.006683437153697014,
+      "rewards//mean": 0.74945068359375,
+      "rewards//std": 0.01489273738116026,
+      "step": 1659
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.332,
+      "grad_norm": 31.912376403808594,
+      "kl": 7.891320111230016,
+      "learning_rate": 7.611816364415895e-07,
+      "loss": 0.3157,
+      "num_tokens": 14347980.0,
+      "reward": 0.72967529296875,
+      "reward_std": 0.02069058082997799,
+      "rewards//mean": 0.72967529296875,
+      "rewards//std": 0.049942903220653534,
+      "step": 1660
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3322,
+      "grad_norm": 11.64772891998291,
+      "kl": 1.9665944166481495,
+      "learning_rate": 7.60910987040623e-07,
+      "loss": 0.0787,
+      "num_tokens": 14356572.0,
+      "reward": 0.76629638671875,
+      "reward_std": 0.005575934890657663,
+      "rewards//mean": 0.76629638671875,
+      "rewards//std": 0.016507720574736595,
+      "step": 1661
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3324,
+      "grad_norm": 16.980592727661133,
+      "kl": 6.588000485673547,
+      "learning_rate": 7.606402325447419e-07,
+      "loss": 0.2635,
+      "num_tokens": 14365332.0,
+      "reward": 0.7310791015625,
+      "reward_std": 0.016998887062072754,
+      "rewards//mean": 0.7310791015625,
+      "rewards//std": 0.03285365551710129,
+      "step": 1662
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3326,
+      "grad_norm": 29.69980812072754,
+      "kl": 7.07955215126276,
+      "learning_rate": 7.603693730630066e-07,
+      "loss": 0.2832,
+      "num_tokens": 14373972.0,
+      "reward": 0.74029541015625,
+      "reward_std": 0.011502914130687714,
+      "rewards//mean": 0.74029541015625,
+      "rewards//std": 0.029667524620890617,
+      "step": 1663
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3328,
+      "grad_norm": 16.89678192138672,
+      "kl": 6.799922564998269,
+      "learning_rate": 7.600984087045186e-07,
+      "loss": 0.272,
+      "num_tokens": 14382628.0,
+      "reward": 0.74896240234375,
+      "reward_std": 0.011954502202570438,
+      "rewards//mean": 0.74896240234375,
+      "rewards//std": 0.03989831358194351,
+      "step": 1664
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.333,
+      "grad_norm": 11.457413673400879,
+      "kl": 2.7482737116515636,
+      "learning_rate": 7.598273395784229e-07,
+      "loss": 0.1099,
+      "num_tokens": 14391236.0,
+      "reward": 0.78363037109375,
+      "reward_std": 0.008042111992835999,
+      "rewards//mean": 0.78363037109375,
+      "rewards//std": 0.028481479734182358,
+      "step": 1665
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3332,
+      "grad_norm": 12.461380958557129,
+      "kl": 4.245392709970474,
+      "learning_rate": 7.59556165793906e-07,
+      "loss": 0.1698,
+      "num_tokens": 14399820.0,
+      "reward": 0.72528076171875,
+      "reward_std": 0.006810404360294342,
+      "rewards//mean": 0.72528076171875,
+      "rewards//std": 0.03248918801546097,
+      "step": 1666
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3334,
+      "grad_norm": 6.8713226318359375,
+      "kl": 3.91130817681551,
+      "learning_rate": 7.592848874601963e-07,
+      "loss": 0.1565,
+      "num_tokens": 14408460.0,
+      "reward": 0.76318359375,
+      "reward_std": 0.012699104845523834,
+      "rewards//mean": 0.76318359375,
+      "rewards//std": 0.03380763530731201,
+      "step": 1667
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3336,
+      "grad_norm": 7.350086212158203,
+      "kl": 5.778000093996525,
+      "learning_rate": 7.590135046865651e-07,
+      "loss": 0.2311,
+      "num_tokens": 14417084.0,
+      "reward": 0.76361083984375,
+      "reward_std": 0.011498469859361649,
+      "rewards//mean": 0.76361083984375,
+      "rewards//std": 0.03844371438026428,
+      "step": 1668
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3338,
+      "grad_norm": 11.982793807983398,
+      "kl": 5.074105879291892,
+      "learning_rate": 7.587420175823252e-07,
+      "loss": 0.203,
+      "num_tokens": 14425684.0,
+      "reward": 0.746337890625,
+      "reward_std": 0.017222050577402115,
+      "rewards//mean": 0.746337890625,
+      "rewards//std": 0.028270291164517403,
+      "step": 1669
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.1875,
+      "epoch": 0.334,
+      "grad_norm": 19.137075424194336,
+      "kl": 3.788899175822735,
+      "learning_rate": 7.584704262568314e-07,
+      "loss": 0.1383,
+      "num_tokens": 14434200.0,
+      "reward": 0.75244140625,
+      "reward_std": 0.008026419207453728,
+      "rewards//mean": 0.75244140625,
+      "rewards//std": 0.026268543675541878,
+      "step": 1670
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3342,
+      "grad_norm": 8.649948120117188,
+      "kl": 2.3480862453579903,
+      "learning_rate": 7.581987308194809e-07,
+      "loss": 0.0939,
+      "num_tokens": 14442848.0,
+      "reward": 0.76666259765625,
+      "reward_std": 0.01028990838676691,
+      "rewards//mean": 0.76666259765625,
+      "rewards//std": 0.028299124911427498,
+      "step": 1671
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3344,
+      "grad_norm": 16.29898452758789,
+      "kl": 5.2496380638331175,
+      "learning_rate": 7.579269313797125e-07,
+      "loss": 0.21,
+      "num_tokens": 14451376.0,
+      "reward": 0.7310791015625,
+      "reward_std": 0.00796603225171566,
+      "rewards//mean": 0.7310791015625,
+      "rewards//std": 0.025640688836574554,
+      "step": 1672
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3346,
+      "grad_norm": 5.378942966461182,
+      "kl": 4.060715341940522,
+      "learning_rate": 7.576550280470071e-07,
+      "loss": 0.1624,
+      "num_tokens": 14460112.0,
+      "reward": 0.76409912109375,
+      "reward_std": 0.01511688157916069,
+      "rewards//mean": 0.76409912109375,
+      "rewards//std": 0.02715814672410488,
+      "step": 1673
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3348,
+      "grad_norm": 9.949033737182617,
+      "kl": 1.6426987908780575,
+      "learning_rate": 7.573830209308872e-07,
+      "loss": 0.0657,
+      "num_tokens": 14468744.0,
+      "reward": 0.7333984375,
+      "reward_std": 0.008231493644416332,
+      "rewards//mean": 0.7333984375,
+      "rewards//std": 0.031117962673306465,
+      "step": 1674
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.335,
+      "grad_norm": 6.0058064460754395,
+      "kl": 2.6325108893215656,
+      "learning_rate": 7.57110910140917e-07,
+      "loss": 0.1053,
+      "num_tokens": 14477368.0,
+      "reward": 0.755126953125,
+      "reward_std": 0.010620414279401302,
+      "rewards//mean": 0.755126953125,
+      "rewards//std": 0.02269059419631958,
+      "step": 1675
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3352,
+      "grad_norm": 6.265772819519043,
+      "kl": 2.351148357614875,
+      "learning_rate": 7.568386957867032e-07,
+      "loss": 0.094,
+      "num_tokens": 14486008.0,
+      "reward": 0.76678466796875,
+      "reward_std": 0.007720914203673601,
+      "rewards//mean": 0.76678466796875,
+      "rewards//std": 0.02256004326045513,
+      "step": 1676
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3354,
+      "grad_norm": 7.135132789611816,
+      "kl": 2.418541956692934,
+      "learning_rate": 7.565663779778933e-07,
+      "loss": 0.0967,
+      "num_tokens": 14494608.0,
+      "reward": 0.74639892578125,
+      "reward_std": 0.0088096484541893,
+      "rewards//mean": 0.74639892578125,
+      "rewards//std": 0.02282288484275341,
+      "step": 1677
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3356,
+      "grad_norm": 16.368267059326172,
+      "kl": 3.6359867565333843,
+      "learning_rate": 7.562939568241771e-07,
+      "loss": 0.1454,
+      "num_tokens": 14503208.0,
+      "reward": 0.76263427734375,
+      "reward_std": 0.010166294872760773,
+      "rewards//mean": 0.76263427734375,
+      "rewards//std": 0.014939420856535435,
+      "step": 1678
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3358,
+      "grad_norm": 5.093266010284424,
+      "kl": 1.491914639249444,
+      "learning_rate": 7.560214324352858e-07,
+      "loss": 0.0597,
+      "num_tokens": 14511840.0,
+      "reward": 0.7373046875,
+      "reward_std": 0.003280874341726303,
+      "rewards//mean": 0.7373046875,
+      "rewards//std": 0.026955686509609222,
+      "step": 1679
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.336,
+      "grad_norm": 8.144636154174805,
+      "kl": 2.80340170674026,
+      "learning_rate": 7.55748804920992e-07,
+      "loss": 0.1121,
+      "num_tokens": 14520544.0,
+      "reward": 0.76519775390625,
+      "reward_std": 0.012464512139558792,
+      "rewards//mean": 0.76519775390625,
+      "rewards//std": 0.03017847053706646,
+      "step": 1680
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3362,
+      "grad_norm": 11.081323623657227,
+      "kl": 6.226683981716633,
+      "learning_rate": 7.554760743911103e-07,
+      "loss": 0.2491,
+      "num_tokens": 14529264.0,
+      "reward": 0.73345947265625,
+      "reward_std": 0.01712070405483246,
+      "rewards//mean": 0.73345947265625,
+      "rewards//std": 0.03184684365987778,
+      "step": 1681
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3364,
+      "grad_norm": 6.9883222579956055,
+      "kl": 2.3179250191897154,
+      "learning_rate": 7.552032409554962e-07,
+      "loss": 0.0927,
+      "num_tokens": 14537864.0,
+      "reward": 0.76922607421875,
+      "reward_std": 0.008112422190606594,
+      "rewards//mean": 0.76922607421875,
+      "rewards//std": 0.020271258428692818,
+      "step": 1682
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3366,
+      "grad_norm": 6.01211404800415,
+      "kl": 2.5319530349224806,
+      "learning_rate": 7.549303047240474e-07,
+      "loss": 0.1013,
+      "num_tokens": 14546472.0,
+      "reward": 0.77532958984375,
+      "reward_std": 0.006156784947961569,
+      "rewards//mean": 0.77532958984375,
+      "rewards//std": 0.03255574777722359,
+      "step": 1683
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3368,
+      "grad_norm": 9.942745208740234,
+      "kl": 2.4515086635947227,
+      "learning_rate": 7.54657265806702e-07,
+      "loss": 0.0981,
+      "num_tokens": 14555080.0,
+      "reward": 0.7755126953125,
+      "reward_std": 0.009586556814610958,
+      "rewards//mean": 0.7755126953125,
+      "rewards//std": 0.02556264027953148,
+      "step": 1684
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.337,
+      "grad_norm": 16.847671508789062,
+      "kl": 4.299041600897908,
+      "learning_rate": 7.543841243134408e-07,
+      "loss": 0.172,
+      "num_tokens": 14563712.0,
+      "reward": 0.7554931640625,
+      "reward_std": 0.013671418651938438,
+      "rewards//mean": 0.7554931640625,
+      "rewards//std": 0.0289964247494936,
+      "step": 1685
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3372,
+      "grad_norm": 11.128751754760742,
+      "kl": 4.528183134272695,
+      "learning_rate": 7.541108803542845e-07,
+      "loss": 0.1811,
+      "num_tokens": 14572320.0,
+      "reward": 0.7733154296875,
+      "reward_std": 0.020395830273628235,
+      "rewards//mean": 0.7733154296875,
+      "rewards//std": 0.031156614422798157,
+      "step": 1686
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3374,
+      "grad_norm": 36.73457336425781,
+      "kl": 7.407161274924874,
+      "learning_rate": 7.538375340392961e-07,
+      "loss": 0.2963,
+      "num_tokens": 14580984.0,
+      "reward": 0.7178955078125,
+      "reward_std": 0.01460280641913414,
+      "rewards//mean": 0.7178955078125,
+      "rewards//std": 0.03694973886013031,
+      "step": 1687
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.703125,
+      "epoch": 0.3376,
+      "grad_norm": 19.612701416015625,
+      "kl": 7.426185131072998,
+      "learning_rate": 7.535640854785791e-07,
+      "loss": 0.2928,
+      "num_tokens": 14589573.0,
+      "reward": 0.7728271484375,
+      "reward_std": 0.02155712991952896,
+      "rewards//mean": 0.7728271484375,
+      "rewards//std": 0.028461076319217682,
+      "step": 1688
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3378,
+      "grad_norm": 16.157751083374023,
+      "kl": 6.999772354960442,
+      "learning_rate": 7.532905347822791e-07,
+      "loss": 0.28,
+      "num_tokens": 14598197.0,
+      "reward": 0.72467041015625,
+      "reward_std": 0.022015413269400597,
+      "rewards//mean": 0.72467041015625,
+      "rewards//std": 0.038431111723184586,
+      "step": 1689
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.338,
+      "grad_norm": 6.553281784057617,
+      "kl": 4.209701828658581,
+      "learning_rate": 7.530168820605818e-07,
+      "loss": 0.1684,
+      "num_tokens": 14606869.0,
+      "reward": 0.74798583984375,
+      "reward_std": 0.012180760502815247,
+      "rewards//mean": 0.74798583984375,
+      "rewards//std": 0.029198473319411278,
+      "step": 1690
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3382,
+      "grad_norm": 32.15467071533203,
+      "kl": 9.1272879447788,
+      "learning_rate": 7.527431274237149e-07,
+      "loss": 0.3651,
+      "num_tokens": 14615597.0,
+      "reward": 0.71746826171875,
+      "reward_std": 0.011943129822611809,
+      "rewards//mean": 0.71746826171875,
+      "rewards//std": 0.028194623067975044,
+      "step": 1691
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3384,
+      "grad_norm": 6.4996209144592285,
+      "kl": 5.02642223611474,
+      "learning_rate": 7.524692709819463e-07,
+      "loss": 0.2011,
+      "num_tokens": 14624213.0,
+      "reward": 0.76025390625,
+      "reward_std": 0.016095131635665894,
+      "rewards//mean": 0.76025390625,
+      "rewards//std": 0.02068750187754631,
+      "step": 1692
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3386,
+      "grad_norm": 11.062028884887695,
+      "kl": 4.206334099173546,
+      "learning_rate": 7.521953128455855e-07,
+      "loss": 0.1683,
+      "num_tokens": 14632813.0,
+      "reward": 0.7607421875,
+      "reward_std": 0.007870189845561981,
+      "rewards//mean": 0.7607421875,
+      "rewards//std": 0.025995122268795967,
+      "step": 1693
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3388,
+      "grad_norm": 11.231415748596191,
+      "kl": 3.808822438120842,
+      "learning_rate": 7.519212531249829e-07,
+      "loss": 0.1524,
+      "num_tokens": 14641413.0,
+      "reward": 0.74603271484375,
+      "reward_std": 0.009955466724932194,
+      "rewards//mean": 0.74603271484375,
+      "rewards//std": 0.022331418469548225,
+      "step": 1694
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.339,
+      "grad_norm": 6.933574676513672,
+      "kl": 2.020120469853282,
+      "learning_rate": 7.516470919305298e-07,
+      "loss": 0.0808,
+      "num_tokens": 14650029.0,
+      "reward": 0.74200439453125,
+      "reward_std": 0.010920392349362373,
+      "rewards//mean": 0.74200439453125,
+      "rewards//std": 0.023688610643148422,
+      "step": 1695
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3392,
+      "grad_norm": 12.914809226989746,
+      "kl": 2.8975869584828615,
+      "learning_rate": 7.513728293726579e-07,
+      "loss": 0.1159,
+      "num_tokens": 14658589.0,
+      "reward": 0.7923583984375,
+      "reward_std": 0.009334616363048553,
+      "rewards//mean": 0.7923583984375,
+      "rewards//std": 0.016619790345430374,
+      "step": 1696
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3394,
+      "grad_norm": 15.476243019104004,
+      "kl": 1.4073762707412243,
+      "learning_rate": 7.510984655618406e-07,
+      "loss": 0.0563,
+      "num_tokens": 14667221.0,
+      "reward": 0.7314453125,
+      "reward_std": 0.0075613269582390785,
+      "rewards//mean": 0.7314453125,
+      "rewards//std": 0.026955686509609222,
+      "step": 1697
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3396,
+      "grad_norm": 15.04116439819336,
+      "kl": 3.4532814882695675,
+      "learning_rate": 7.508240006085913e-07,
+      "loss": 0.1381,
+      "num_tokens": 14675909.0,
+      "reward": 0.755615234375,
+      "reward_std": 0.0094823706895113,
+      "rewards//mean": 0.755615234375,
+      "rewards//std": 0.018040519207715988,
+      "step": 1698
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3398,
+      "grad_norm": 10.518776893615723,
+      "kl": 2.346025651320815,
+      "learning_rate": 7.505494346234647e-07,
+      "loss": 0.0938,
+      "num_tokens": 14684477.0,
+      "reward": 0.76666259765625,
+      "reward_std": 0.011838393285870552,
+      "rewards//mean": 0.76666259765625,
+      "rewards//std": 0.024697864428162575,
+      "step": 1699
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.34,
+      "grad_norm": 13.792692184448242,
+      "kl": 6.545206168666482,
+      "learning_rate": 7.502747677170555e-07,
+      "loss": 0.2618,
+      "num_tokens": 14693117.0,
+      "reward": 0.75,
+      "reward_std": 0.01461731269955635,
+      "rewards//mean": 0.75,
+      "rewards//std": 0.029181944206357002,
+      "step": 1700
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3402,
+      "grad_norm": 8.20025634765625,
+      "kl": 4.336450120434165,
+      "learning_rate": 7.5e-07,
+      "loss": 0.1735,
+      "num_tokens": 14701757.0,
+      "reward": 0.7445068359375,
+      "reward_std": 0.013547065667808056,
+      "rewards//mean": 0.7445068359375,
+      "rewards//std": 0.03472801297903061,
+      "step": 1701
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3404,
+      "grad_norm": 13.76013469696045,
+      "kl": 4.508783459663391,
+      "learning_rate": 7.497251315829743e-07,
+      "loss": 0.1804,
+      "num_tokens": 14710437.0,
+      "reward": 0.75933837890625,
+      "reward_std": 0.008789284154772758,
+      "rewards//mean": 0.75933837890625,
+      "rewards//std": 0.01940830983221531,
+      "step": 1702
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3406,
+      "grad_norm": 14.223562240600586,
+      "kl": 5.793676137924194,
+      "learning_rate": 7.494501625766955e-07,
+      "loss": 0.2317,
+      "num_tokens": 14719157.0,
+      "reward": 0.76434326171875,
+      "reward_std": 0.011211156845092773,
+      "rewards//mean": 0.76434326171875,
+      "rewards//std": 0.028211798518896103,
+      "step": 1703
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3408,
+      "grad_norm": 7.0896711349487305,
+      "kl": 4.318609192967415,
+      "learning_rate": 7.491750930919212e-07,
+      "loss": 0.1727,
+      "num_tokens": 14727781.0,
+      "reward": 0.76995849609375,
+      "reward_std": 0.00895887054502964,
+      "rewards//mean": 0.76995849609375,
+      "rewards//std": 0.02394981123507023,
+      "step": 1704
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.341,
+      "grad_norm": 12.570860862731934,
+      "kl": 4.320924017578363,
+      "learning_rate": 7.488999232394491e-07,
+      "loss": 0.1728,
+      "num_tokens": 14736469.0,
+      "reward": 0.75244140625,
+      "reward_std": 0.006509346887469292,
+      "rewards//mean": 0.75244140625,
+      "rewards//std": 0.024324923753738403,
+      "step": 1705
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3412,
+      "grad_norm": 16.738834381103516,
+      "kl": 2.4809902161359787,
+      "learning_rate": 7.486246531301177e-07,
+      "loss": 0.0992,
+      "num_tokens": 14745133.0,
+      "reward": 0.72711181640625,
+      "reward_std": 0.013168182224035263,
+      "rewards//mean": 0.72711181640625,
+      "rewards//std": 0.03143443912267685,
+      "step": 1706
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.46875,
+      "epoch": 0.3414,
+      "grad_norm": 6.93234395980835,
+      "kl": 5.587183387950063,
+      "learning_rate": 7.483492828748056e-07,
+      "loss": 0.2142,
+      "num_tokens": 14753827.0,
+      "reward": 0.7552490234375,
+      "reward_std": 0.016453320160508156,
+      "rewards//mean": 0.7552490234375,
+      "rewards//std": 0.02872367389500141,
+      "step": 1707
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3416,
+      "grad_norm": 22.446189880371094,
+      "kl": 2.741853207349777,
+      "learning_rate": 7.480738125844322e-07,
+      "loss": 0.1097,
+      "num_tokens": 14762443.0,
+      "reward": 0.73297119140625,
+      "reward_std": 0.011288801208138466,
+      "rewards//mean": 0.73297119140625,
+      "rewards//std": 0.030681896954774857,
+      "step": 1708
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3418,
+      "grad_norm": 12.996790885925293,
+      "kl": 2.3409229721874,
+      "learning_rate": 7.477982423699567e-07,
+      "loss": 0.0936,
+      "num_tokens": 14771099.0,
+      "reward": 0.772705078125,
+      "reward_std": 0.008740413933992386,
+      "rewards//mean": 0.772705078125,
+      "rewards//std": 0.01942399889230728,
+      "step": 1709
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.342,
+      "grad_norm": 8.097086906433105,
+      "kl": 1.8513435777276754,
+      "learning_rate": 7.475225723423788e-07,
+      "loss": 0.0741,
+      "num_tokens": 14779779.0,
+      "reward": 0.7666015625,
+      "reward_std": 0.007709752302616835,
+      "rewards//mean": 0.7666015625,
+      "rewards//std": 0.021202094852924347,
+      "step": 1710
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3422,
+      "grad_norm": 12.267953872680664,
+      "kl": 5.723224747925997,
+      "learning_rate": 7.472468026127384e-07,
+      "loss": 0.2289,
+      "num_tokens": 14788555.0,
+      "reward": 0.76629638671875,
+      "reward_std": 0.009873709641397,
+      "rewards//mean": 0.76629638671875,
+      "rewards//std": 0.02834455668926239,
+      "step": 1711
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3424,
+      "grad_norm": 4.721216201782227,
+      "kl": 4.228861682116985,
+      "learning_rate": 7.469709332921154e-07,
+      "loss": 0.1692,
+      "num_tokens": 14797115.0,
+      "reward": 0.73907470703125,
+      "reward_std": 0.011778753250837326,
+      "rewards//mean": 0.73907470703125,
+      "rewards//std": 0.027945473790168762,
+      "step": 1712
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3426,
+      "grad_norm": 7.927215576171875,
+      "kl": 4.117089059203863,
+      "learning_rate": 7.4669496449163e-07,
+      "loss": 0.1647,
+      "num_tokens": 14805763.0,
+      "reward": 0.74078369140625,
+      "reward_std": 0.01242013182491064,
+      "rewards//mean": 0.74078369140625,
+      "rewards//std": 0.020825976505875587,
+      "step": 1713
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3428,
+      "grad_norm": 16.150463104248047,
+      "kl": 6.161867793649435,
+      "learning_rate": 7.464188963224427e-07,
+      "loss": 0.2465,
+      "num_tokens": 14814371.0,
+      "reward": 0.75897216796875,
+      "reward_std": 0.011923184618353844,
+      "rewards//mean": 0.75897216796875,
+      "rewards//std": 0.03031909465789795,
+      "step": 1714
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.734375,
+      "epoch": 0.343,
+      "grad_norm": 8.781957626342773,
+      "kl": 3.492230284959078,
+      "learning_rate": 7.461427288957531e-07,
+      "loss": 0.1432,
+      "num_tokens": 14822954.0,
+      "reward": 0.74908447265625,
+      "reward_std": 0.008261223323643208,
+      "rewards//mean": 0.74908447265625,
+      "rewards//std": 0.02625233121216297,
+      "step": 1715
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3432,
+      "grad_norm": 17.5076961517334,
+      "kl": 3.583744313567877,
+      "learning_rate": 7.45866462322802e-07,
+      "loss": 0.1433,
+      "num_tokens": 14831530.0,
+      "reward": 0.7747802734375,
+      "reward_std": 0.009402611292898655,
+      "rewards//mean": 0.7747802734375,
+      "rewards//std": 0.019872011616826057,
+      "step": 1716
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3434,
+      "grad_norm": 11.033143997192383,
+      "kl": 5.401227429509163,
+      "learning_rate": 7.45590096714869e-07,
+      "loss": 0.216,
+      "num_tokens": 14840114.0,
+      "reward": 0.72332763671875,
+      "reward_std": 0.020782142877578735,
+      "rewards//mean": 0.72332763671875,
+      "rewards//std": 0.029228009283542633,
+      "step": 1717
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3436,
+      "grad_norm": 16.427082061767578,
+      "kl": 6.726500168442726,
+      "learning_rate": 7.453136321832745e-07,
+      "loss": 0.2691,
+      "num_tokens": 14848722.0,
+      "reward": 0.760009765625,
+      "reward_std": 0.012372981756925583,
+      "rewards//mean": 0.760009765625,
+      "rewards//std": 0.021473098546266556,
+      "step": 1718
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3438,
+      "grad_norm": 26.458110809326172,
+      "kl": 7.284464478492737,
+      "learning_rate": 7.450370688393784e-07,
+      "loss": 0.2914,
+      "num_tokens": 14857322.0,
+      "reward": 0.73175048828125,
+      "reward_std": 0.012284617871046066,
+      "rewards//mean": 0.73175048828125,
+      "rewards//std": 0.02642074227333069,
+      "step": 1719
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.344,
+      "grad_norm": 13.820018768310547,
+      "kl": 5.9415582139045,
+      "learning_rate": 7.447604067945802e-07,
+      "loss": 0.2377,
+      "num_tokens": 14866018.0,
+      "reward": 0.72735595703125,
+      "reward_std": 0.013435333967208862,
+      "rewards//mean": 0.72735595703125,
+      "rewards//std": 0.0270917359739542,
+      "step": 1720
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3442,
+      "grad_norm": 14.459245681762695,
+      "kl": 5.953585060313344,
+      "learning_rate": 7.444836461603194e-07,
+      "loss": 0.2381,
+      "num_tokens": 14874850.0,
+      "reward": 0.75592041015625,
+      "reward_std": 0.011368580162525177,
+      "rewards//mean": 0.75592041015625,
+      "rewards//std": 0.024279998615384102,
+      "step": 1721
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3444,
+      "grad_norm": 11.407780647277832,
+      "kl": 5.483324466273189,
+      "learning_rate": 7.442067870480751e-07,
+      "loss": 0.2193,
+      "num_tokens": 14883418.0,
+      "reward": 0.7259521484375,
+      "reward_std": 0.009960510767996311,
+      "rewards//mean": 0.7259521484375,
+      "rewards//std": 0.03520938754081726,
+      "step": 1722
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3446,
+      "grad_norm": 10.866833686828613,
+      "kl": 4.219879038631916,
+      "learning_rate": 7.439298295693663e-07,
+      "loss": 0.1688,
+      "num_tokens": 14892090.0,
+      "reward": 0.7445068359375,
+      "reward_std": 0.006772883702069521,
+      "rewards//mean": 0.7445068359375,
+      "rewards//std": 0.03010294772684574,
+      "step": 1723
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3448,
+      "grad_norm": 36.98476791381836,
+      "kl": 3.116904040798545,
+      "learning_rate": 7.436527738357513e-07,
+      "loss": 0.1247,
+      "num_tokens": 14900690.0,
+      "reward": 0.76971435546875,
+      "reward_std": 0.01532891858369112,
+      "rewards//mean": 0.76971435546875,
+      "rewards//std": 0.03278140723705292,
+      "step": 1724
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.345,
+      "grad_norm": 5.003154754638672,
+      "kl": 2.0285655725747347,
+      "learning_rate": 7.433756199588282e-07,
+      "loss": 0.0811,
+      "num_tokens": 14909338.0,
+      "reward": 0.7744140625,
+      "reward_std": 0.004872136749327183,
+      "rewards//mean": 0.7744140625,
+      "rewards//std": 0.02111050859093666,
+      "step": 1725
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3452,
+      "grad_norm": 18.3594913482666,
+      "kl": 1.8213788475841284,
+      "learning_rate": 7.430983680502343e-07,
+      "loss": 0.0729,
+      "num_tokens": 14918018.0,
+      "reward": 0.75994873046875,
+      "reward_std": 0.00850547756999731,
+      "rewards//mean": 0.75994873046875,
+      "rewards//std": 0.02573225647211075,
+      "step": 1726
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3454,
+      "grad_norm": 6.738495826721191,
+      "kl": 1.195437179878354,
+      "learning_rate": 7.42821018221647e-07,
+      "loss": 0.0478,
+      "num_tokens": 14926762.0,
+      "reward": 0.78759765625,
+      "reward_std": 0.003485372057184577,
+      "rewards//mean": 0.78759765625,
+      "rewards//std": 0.02826707810163498,
+      "step": 1727
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3456,
+      "grad_norm": 7.730818271636963,
+      "kl": 2.8814690820872784,
+      "learning_rate": 7.425435705847825e-07,
+      "loss": 0.1153,
+      "num_tokens": 14935474.0,
+      "reward": 0.7613525390625,
+      "reward_std": 0.010984411463141441,
+      "rewards//mean": 0.7613525390625,
+      "rewards//std": 0.01968218944966793,
+      "step": 1728
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3458,
+      "grad_norm": 10.770354270935059,
+      "kl": 2.826941965147853,
+      "learning_rate": 7.422660252513968e-07,
+      "loss": 0.1131,
+      "num_tokens": 14944122.0,
+      "reward": 0.75079345703125,
+      "reward_std": 0.010136321187019348,
+      "rewards//mean": 0.75079345703125,
+      "rewards//std": 0.021352121606469154,
+      "step": 1729
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.346,
+      "grad_norm": 8.526028633117676,
+      "kl": 1.2991268746554852,
+      "learning_rate": 7.41988382333285e-07,
+      "loss": 0.052,
+      "num_tokens": 14952810.0,
+      "reward": 0.74993896484375,
+      "reward_std": 0.00819031335413456,
+      "rewards//mean": 0.74993896484375,
+      "rewards//std": 0.02521466091275215,
+      "step": 1730
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3462,
+      "grad_norm": 6.339461803436279,
+      "kl": 1.9494109824299812,
+      "learning_rate": 7.417106419422818e-07,
+      "loss": 0.078,
+      "num_tokens": 14961378.0,
+      "reward": 0.7952880859375,
+      "reward_std": 0.0145040825009346,
+      "rewards//mean": 0.7952880859375,
+      "rewards//std": 0.027425285428762436,
+      "step": 1731
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3464,
+      "grad_norm": 19.363574981689453,
+      "kl": 1.2932778727263212,
+      "learning_rate": 7.41432804190261e-07,
+      "loss": 0.0517,
+      "num_tokens": 14969962.0,
+      "reward": 0.72003173828125,
+      "reward_std": 0.003600679337978363,
+      "rewards//mean": 0.72003173828125,
+      "rewards//std": 0.027648132294416428,
+      "step": 1732
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3466,
+      "grad_norm": 6.703650951385498,
+      "kl": 3.404814265668392,
+      "learning_rate": 7.411548691891357e-07,
+      "loss": 0.1362,
+      "num_tokens": 14978610.0,
+      "reward": 0.7841796875,
+      "reward_std": 0.017170464619994164,
+      "rewards//mean": 0.7841796875,
+      "rewards//std": 0.03392563387751579,
+      "step": 1733
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3468,
+      "grad_norm": 4.404158592224121,
+      "kl": 2.9254848584532738,
+      "learning_rate": 7.408768370508576e-07,
+      "loss": 0.117,
+      "num_tokens": 14987322.0,
+      "reward": 0.76251220703125,
+      "reward_std": 0.01290039625018835,
+      "rewards//mean": 0.76251220703125,
+      "rewards//std": 0.028272902593016624,
+      "step": 1734
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.347,
+      "grad_norm": 22.63078498840332,
+      "kl": 2.2710730005055666,
+      "learning_rate": 7.405987078874185e-07,
+      "loss": 0.0908,
+      "num_tokens": 14995930.0,
+      "reward": 0.7584228515625,
+      "reward_std": 0.012374240905046463,
+      "rewards//mean": 0.7584228515625,
+      "rewards//std": 0.03130590170621872,
+      "step": 1735
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3472,
+      "grad_norm": 13.92486572265625,
+      "kl": 2.887278689071536,
+      "learning_rate": 7.403204818108487e-07,
+      "loss": 0.1155,
+      "num_tokens": 15004594.0,
+      "reward": 0.7598876953125,
+      "reward_std": 0.011876486241817474,
+      "rewards//mean": 0.7598876953125,
+      "rewards//std": 0.03911187872290611,
+      "step": 1736
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3474,
+      "grad_norm": 7.4634690284729,
+      "kl": 2.634060813114047,
+      "learning_rate": 7.400421589332174e-07,
+      "loss": 0.1054,
+      "num_tokens": 15013186.0,
+      "reward": 0.76824951171875,
+      "reward_std": 0.00886482559144497,
+      "rewards//mean": 0.76824951171875,
+      "rewards//std": 0.0291574876755476,
+      "step": 1737
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3476,
+      "grad_norm": 8.981084823608398,
+      "kl": 4.878067035228014,
+      "learning_rate": 7.397637393666333e-07,
+      "loss": 0.1951,
+      "num_tokens": 15021962.0,
+      "reward": 0.7420654296875,
+      "reward_std": 0.01109348889440298,
+      "rewards//mean": 0.7420654296875,
+      "rewards//std": 0.024269787594676018,
+      "step": 1738
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3478,
+      "grad_norm": 14.383384704589844,
+      "kl": 2.241973152384162,
+      "learning_rate": 7.394852232232436e-07,
+      "loss": 0.0897,
+      "num_tokens": 15030642.0,
+      "reward": 0.79779052734375,
+      "reward_std": 0.011168712750077248,
+      "rewards//mean": 0.79779052734375,
+      "rewards//std": 0.02340381033718586,
+      "step": 1739
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.348,
+      "grad_norm": 7.720381736755371,
+      "kl": 3.893526293337345,
+      "learning_rate": 7.392066106152345e-07,
+      "loss": 0.1557,
+      "num_tokens": 15039298.0,
+      "reward": 0.7689208984375,
+      "reward_std": 0.008886430412530899,
+      "rewards//mean": 0.7689208984375,
+      "rewards//std": 0.025546055287122726,
+      "step": 1740
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3482,
+      "grad_norm": 5.949435710906982,
+      "kl": 2.1611736863851547,
+      "learning_rate": 7.389279016548316e-07,
+      "loss": 0.0864,
+      "num_tokens": 15047898.0,
+      "reward": 0.75262451171875,
+      "reward_std": 0.008822827599942684,
+      "rewards//mean": 0.75262451171875,
+      "rewards//std": 0.028772840276360512,
+      "step": 1741
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3484,
+      "grad_norm": 3.424663543701172,
+      "kl": 2.818216495215893,
+      "learning_rate": 7.386490964542982e-07,
+      "loss": 0.1127,
+      "num_tokens": 15056522.0,
+      "reward": 0.78424072265625,
+      "reward_std": 0.009205842390656471,
+      "rewards//mean": 0.78424072265625,
+      "rewards//std": 0.02906336635351181,
+      "step": 1742
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3486,
+      "grad_norm": 17.820810317993164,
+      "kl": 2.490190641954541,
+      "learning_rate": 7.383701951259375e-07,
+      "loss": 0.0996,
+      "num_tokens": 15065074.0,
+      "reward": 0.7767333984375,
+      "reward_std": 0.006568721029907465,
+      "rewards//mean": 0.7767333984375,
+      "rewards//std": 0.02129577472805977,
+      "step": 1743
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3488,
+      "grad_norm": 9.225910186767578,
+      "kl": 5.337249953299761,
+      "learning_rate": 7.380911977820906e-07,
+      "loss": 0.2135,
+      "num_tokens": 15073762.0,
+      "reward": 0.7352294921875,
+      "reward_std": 0.015421277843415737,
+      "rewards//mean": 0.7352294921875,
+      "rewards//std": 0.024304691702127457,
+      "step": 1744
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.349,
+      "grad_norm": 21.589153289794922,
+      "kl": 6.036288775503635,
+      "learning_rate": 7.378121045351377e-07,
+      "loss": 0.2415,
+      "num_tokens": 15082394.0,
+      "reward": 0.744873046875,
+      "reward_std": 0.010433953255414963,
+      "rewards//mean": 0.744873046875,
+      "rewards//std": 0.023165950551629066,
+      "step": 1745
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3492,
+      "grad_norm": 11.4340181350708,
+      "kl": 1.4951294343918562,
+      "learning_rate": 7.375329154974975e-07,
+      "loss": 0.0598,
+      "num_tokens": 15091026.0,
+      "reward": 0.7608642578125,
+      "reward_std": 0.005236084572970867,
+      "rewards//mean": 0.7608642578125,
+      "rewards//std": 0.02316170372068882,
+      "step": 1746
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3494,
+      "grad_norm": 12.729737281799316,
+      "kl": 5.8137662298977375,
+      "learning_rate": 7.372536307816272e-07,
+      "loss": 0.2326,
+      "num_tokens": 15099618.0,
+      "reward": 0.759765625,
+      "reward_std": 0.010998114943504333,
+      "rewards//mean": 0.759765625,
+      "rewards//std": 0.01820257492363453,
+      "step": 1747
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3496,
+      "grad_norm": 15.836647033691406,
+      "kl": 5.377154625952244,
+      "learning_rate": 7.369742505000231e-07,
+      "loss": 0.2151,
+      "num_tokens": 15108242.0,
+      "reward": 0.75054931640625,
+      "reward_std": 0.010375864803791046,
+      "rewards//mean": 0.75054931640625,
+      "rewards//std": 0.036457452923059464,
+      "step": 1748
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3498,
+      "grad_norm": 28.29893684387207,
+      "kl": 6.580530092120171,
+      "learning_rate": 7.366947747652191e-07,
+      "loss": 0.2632,
+      "num_tokens": 15116810.0,
+      "reward": 0.72918701171875,
+      "reward_std": 0.01071253977715969,
+      "rewards//mean": 0.72918701171875,
+      "rewards//std": 0.026929209008812904,
+      "step": 1749
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.35,
+      "grad_norm": 19.780168533325195,
+      "kl": 7.418893063440919,
+      "learning_rate": 7.364152036897882e-07,
+      "loss": 0.2968,
+      "num_tokens": 15125418.0,
+      "reward": 0.7392578125,
+      "reward_std": 0.013927556574344635,
+      "rewards//mean": 0.7392578125,
+      "rewards//std": 0.03685477748513222,
+      "step": 1750
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3502,
+      "grad_norm": 14.399723052978516,
+      "kl": 6.29911071062088,
+      "learning_rate": 7.361355373863413e-07,
+      "loss": 0.252,
+      "num_tokens": 15133986.0,
+      "reward": 0.73876953125,
+      "reward_std": 0.011047488078474998,
+      "rewards//mean": 0.73876953125,
+      "rewards//std": 0.03426307067275047,
+      "step": 1751
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3504,
+      "grad_norm": 17.051923751831055,
+      "kl": 2.556887049227953,
+      "learning_rate": 7.358557759675284e-07,
+      "loss": 0.1023,
+      "num_tokens": 15142714.0,
+      "reward": 0.7666015625,
+      "reward_std": 0.01128496415913105,
+      "rewards//mean": 0.7666015625,
+      "rewards//std": 0.02542041800916195,
+      "step": 1752
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3506,
+      "grad_norm": 11.291234016418457,
+      "kl": 4.278929254040122,
+      "learning_rate": 7.35575919546037e-07,
+      "loss": 0.1712,
+      "num_tokens": 15151538.0,
+      "reward": 0.7235107421875,
+      "reward_std": 0.010619303211569786,
+      "rewards//mean": 0.7235107421875,
+      "rewards//std": 0.03514914587140083,
+      "step": 1753
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3508,
+      "grad_norm": 5.1102423667907715,
+      "kl": 2.5491420943289995,
+      "learning_rate": 7.352959682345935e-07,
+      "loss": 0.102,
+      "num_tokens": 15160170.0,
+      "reward": 0.7515869140625,
+      "reward_std": 0.00870327465236187,
+      "rewards//mean": 0.7515869140625,
+      "rewards//std": 0.020795151591300964,
+      "step": 1754
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.351,
+      "grad_norm": 8.712926864624023,
+      "kl": 3.3026189766824245,
+      "learning_rate": 7.350159221459621e-07,
+      "loss": 0.1321,
+      "num_tokens": 15168810.0,
+      "reward": 0.7755126953125,
+      "reward_std": 0.009872986003756523,
+      "rewards//mean": 0.7755126953125,
+      "rewards//std": 0.022988514974713326,
+      "step": 1755
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3512,
+      "grad_norm": 5.928198337554932,
+      "kl": 4.434307049959898,
+      "learning_rate": 7.347357813929454e-07,
+      "loss": 0.1774,
+      "num_tokens": 15177346.0,
+      "reward": 0.75616455078125,
+      "reward_std": 0.010966308414936066,
+      "rewards//mean": 0.75616455078125,
+      "rewards//std": 0.02040819823741913,
+      "step": 1756
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3514,
+      "grad_norm": 9.019213676452637,
+      "kl": 3.6429110188037157,
+      "learning_rate": 7.344555460883839e-07,
+      "loss": 0.1457,
+      "num_tokens": 15185914.0,
+      "reward": 0.76507568359375,
+      "reward_std": 0.011363131925463676,
+      "rewards//mean": 0.76507568359375,
+      "rewards//std": 0.019745443016290665,
+      "step": 1757
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3516,
+      "grad_norm": 3.725701332092285,
+      "kl": 2.5000802017748356,
+      "learning_rate": 7.341752163451567e-07,
+      "loss": 0.1,
+      "num_tokens": 15194562.0,
+      "reward": 0.73394775390625,
+      "reward_std": 0.0064627304673194885,
+      "rewards//mean": 0.73394775390625,
+      "rewards//std": 0.027233846485614777,
+      "step": 1758
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3518,
+      "grad_norm": 7.178076267242432,
+      "kl": 2.457119194790721,
+      "learning_rate": 7.338947922761802e-07,
+      "loss": 0.0983,
+      "num_tokens": 15203194.0,
+      "reward": 0.75592041015625,
+      "reward_std": 0.00877198576927185,
+      "rewards//mean": 0.75592041015625,
+      "rewards//std": 0.024230070412158966,
+      "step": 1759
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.90625,
+      "epoch": 0.352,
+      "grad_norm": 5.549074649810791,
+      "kl": 3.2554349713027477,
+      "learning_rate": 7.336142739944093e-07,
+      "loss": 0.1287,
+      "num_tokens": 15211908.0,
+      "reward": 0.7672119140625,
+      "reward_std": 0.011690272949635983,
+      "rewards//mean": 0.7672119140625,
+      "rewards//std": 0.029046498239040375,
+      "step": 1760
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3522,
+      "grad_norm": 10.549675941467285,
+      "kl": 1.002486851066351,
+      "learning_rate": 7.333336616128369e-07,
+      "loss": 0.0401,
+      "num_tokens": 15220524.0,
+      "reward": 0.76007080078125,
+      "reward_std": 0.003659485839307308,
+      "rewards//mean": 0.76007080078125,
+      "rewards//std": 0.01995212212204933,
+      "step": 1761
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3524,
+      "grad_norm": 10.45427417755127,
+      "kl": 2.251881780102849,
+      "learning_rate": 7.330529552444932e-07,
+      "loss": 0.0901,
+      "num_tokens": 15229196.0,
+      "reward": 0.74853515625,
+      "reward_std": 0.006151488516479731,
+      "rewards//mean": 0.74853515625,
+      "rewards//std": 0.022654540836811066,
+      "step": 1762
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3526,
+      "grad_norm": 15.576757431030273,
+      "kl": 2.9452833347022533,
+      "learning_rate": 7.327721550024475e-07,
+      "loss": 0.1178,
+      "num_tokens": 15237892.0,
+      "reward": 0.74267578125,
+      "reward_std": 0.010885363444685936,
+      "rewards//mean": 0.74267578125,
+      "rewards//std": 0.026707449927926064,
+      "step": 1763
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3528,
+      "grad_norm": 5.1858415603637695,
+      "kl": 1.606558632105589,
+      "learning_rate": 7.324912609998053e-07,
+      "loss": 0.0643,
+      "num_tokens": 15246564.0,
+      "reward": 0.733154296875,
+      "reward_std": 0.004549759905785322,
+      "rewards//mean": 0.733154296875,
+      "rewards//std": 0.025189509615302086,
+      "step": 1764
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.353,
+      "grad_norm": 9.590002059936523,
+      "kl": 4.183308666571975,
+      "learning_rate": 7.322102733497109e-07,
+      "loss": 0.1673,
+      "num_tokens": 15255244.0,
+      "reward": 0.770263671875,
+      "reward_std": 0.00666767219081521,
+      "rewards//mean": 0.770263671875,
+      "rewards//std": 0.031938981264829636,
+      "step": 1765
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3532,
+      "grad_norm": 10.281412124633789,
+      "kl": 3.2724026404321194,
+      "learning_rate": 7.319291921653463e-07,
+      "loss": 0.1309,
+      "num_tokens": 15263876.0,
+      "reward": 0.73870849609375,
+      "reward_std": 0.004603174515068531,
+      "rewards//mean": 0.73870849609375,
+      "rewards//std": 0.022968998178839684,
+      "step": 1766
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3534,
+      "grad_norm": 8.301042556762695,
+      "kl": 5.278777930885553,
+      "learning_rate": 7.316480175599308e-07,
+      "loss": 0.2112,
+      "num_tokens": 15272540.0,
+      "reward": 0.78741455078125,
+      "reward_std": 0.015570227056741714,
+      "rewards//mean": 0.78741455078125,
+      "rewards//std": 0.02587188594043255,
+      "step": 1767
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3536,
+      "grad_norm": 9.332090377807617,
+      "kl": 2.2536631040275097,
+      "learning_rate": 7.313667496467215e-07,
+      "loss": 0.0901,
+      "num_tokens": 15281124.0,
+      "reward": 0.74981689453125,
+      "reward_std": 0.005827593617141247,
+      "rewards//mean": 0.74981689453125,
+      "rewards//std": 0.014506562612950802,
+      "step": 1768
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 126.765625,
+      "epoch": 0.3538,
+      "grad_norm": 3.0023088455200195,
+      "kl": 2.0433195158839226,
+      "learning_rate": 7.310853885390132e-07,
+      "loss": 0.0576,
+      "num_tokens": 15289653.0,
+      "reward": 0.78173828125,
+      "reward_std": 0.005920770578086376,
+      "rewards//mean": 0.78173828125,
+      "rewards//std": 0.01923132874071598,
+      "step": 1769
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.354,
+      "grad_norm": 7.975004196166992,
+      "kl": 5.059855833649635,
+      "learning_rate": 7.308039343501379e-07,
+      "loss": 0.2024,
+      "num_tokens": 15298357.0,
+      "reward": 0.72723388671875,
+      "reward_std": 0.009100313298404217,
+      "rewards//mean": 0.72723388671875,
+      "rewards//std": 0.028608225286006927,
+      "step": 1770
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3542,
+      "grad_norm": 10.475563049316406,
+      "kl": 2.0413386076688766,
+      "learning_rate": 7.305223871934656e-07,
+      "loss": 0.0817,
+      "num_tokens": 15307029.0,
+      "reward": 0.7882080078125,
+      "reward_std": 0.0049152253195643425,
+      "rewards//mean": 0.7882080078125,
+      "rewards//std": 0.01619539223611355,
+      "step": 1771
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3544,
+      "grad_norm": 16.09853744506836,
+      "kl": 4.93830094113946,
+      "learning_rate": 7.302407471824033e-07,
+      "loss": 0.1975,
+      "num_tokens": 15315717.0,
+      "reward": 0.7646484375,
+      "reward_std": 0.007313862442970276,
+      "rewards//mean": 0.7646484375,
+      "rewards//std": 0.03219672292470932,
+      "step": 1772
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3546,
+      "grad_norm": 3.7485849857330322,
+      "kl": 2.3264099545776844,
+      "learning_rate": 7.299590144303954e-07,
+      "loss": 0.0931,
+      "num_tokens": 15324325.0,
+      "reward": 0.76885986328125,
+      "reward_std": 0.007083921227604151,
+      "rewards//mean": 0.76885986328125,
+      "rewards//std": 0.02356688305735588,
+      "step": 1773
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3548,
+      "grad_norm": 12.827655792236328,
+      "kl": 3.729407176375389,
+      "learning_rate": 7.296771890509242e-07,
+      "loss": 0.1492,
+      "num_tokens": 15332901.0,
+      "reward": 0.76171875,
+      "reward_std": 0.004367573652416468,
+      "rewards//mean": 0.76171875,
+      "rewards//std": 0.025581879541277885,
+      "step": 1774
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.355,
+      "grad_norm": 10.825053215026855,
+      "kl": 8.491058219224215,
+      "learning_rate": 7.293952711575086e-07,
+      "loss": 0.3396,
+      "num_tokens": 15341501.0,
+      "reward": 0.74847412109375,
+      "reward_std": 0.01544426754117012,
+      "rewards//mean": 0.74847412109375,
+      "rewards//std": 0.03469241410493851,
+      "step": 1775
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3552,
+      "grad_norm": 12.783354759216309,
+      "kl": 3.17761186696589,
+      "learning_rate": 7.291132608637052e-07,
+      "loss": 0.1271,
+      "num_tokens": 15350125.0,
+      "reward": 0.78692626953125,
+      "reward_std": 0.009350663051009178,
+      "rewards//mean": 0.78692626953125,
+      "rewards//std": 0.02011081762611866,
+      "step": 1776
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3554,
+      "grad_norm": 8.209219932556152,
+      "kl": 3.952581450343132,
+      "learning_rate": 7.288311582831077e-07,
+      "loss": 0.1581,
+      "num_tokens": 15358829.0,
+      "reward": 0.77197265625,
+      "reward_std": 0.006296072620898485,
+      "rewards//mean": 0.77197265625,
+      "rewards//std": 0.02374039590358734,
+      "step": 1777
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3556,
+      "grad_norm": 19.983972549438477,
+      "kl": 9.637716632336378,
+      "learning_rate": 7.285489635293471e-07,
+      "loss": 0.3855,
+      "num_tokens": 15367453.0,
+      "reward": 0.74346923828125,
+      "reward_std": 0.018547261133790016,
+      "rewards//mean": 0.74346923828125,
+      "rewards//std": 0.02717096358537674,
+      "step": 1778
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3558,
+      "grad_norm": 22.738313674926758,
+      "kl": 7.230593521147966,
+      "learning_rate": 7.282666767160912e-07,
+      "loss": 0.2892,
+      "num_tokens": 15376125.0,
+      "reward": 0.7393798828125,
+      "reward_std": 0.008969064801931381,
+      "rewards//mean": 0.7393798828125,
+      "rewards//std": 0.02510855160653591,
+      "step": 1779
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.356,
+      "grad_norm": 25.0104923248291,
+      "kl": 5.67492107860744,
+      "learning_rate": 7.279842979570453e-07,
+      "loss": 0.227,
+      "num_tokens": 15384829.0,
+      "reward": 0.7806396484375,
+      "reward_std": 0.004135373514145613,
+      "rewards//mean": 0.7806396484375,
+      "rewards//std": 0.018125487491488457,
+      "step": 1780
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3562,
+      "grad_norm": 6.182112216949463,
+      "kl": 3.115319015458226,
+      "learning_rate": 7.277018273659516e-07,
+      "loss": 0.1246,
+      "num_tokens": 15393437.0,
+      "reward": 0.78009033203125,
+      "reward_std": 0.005007099360227585,
+      "rewards//mean": 0.78009033203125,
+      "rewards//std": 0.027389591559767723,
+      "step": 1781
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3564,
+      "grad_norm": 27.994529724121094,
+      "kl": 5.9446770660579205,
+      "learning_rate": 7.274192650565889e-07,
+      "loss": 0.2378,
+      "num_tokens": 15402053.0,
+      "reward": 0.7587890625,
+      "reward_std": 0.011876787059009075,
+      "rewards//mean": 0.7587890625,
+      "rewards//std": 0.02433985471725464,
+      "step": 1782
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3566,
+      "grad_norm": 28.30736541748047,
+      "kl": 9.795169910416007,
+      "learning_rate": 7.271366111427734e-07,
+      "loss": 0.3918,
+      "num_tokens": 15410757.0,
+      "reward": 0.7508544921875,
+      "reward_std": 0.01354462280869484,
+      "rewards//mean": 0.7508544921875,
+      "rewards//std": 0.03079301305115223,
+      "step": 1783
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3568,
+      "grad_norm": 11.573566436767578,
+      "kl": 2.8760644625872374,
+      "learning_rate": 7.26853865738358e-07,
+      "loss": 0.115,
+      "num_tokens": 15419405.0,
+      "reward": 0.751953125,
+      "reward_std": 0.004005537834018469,
+      "rewards//mean": 0.751953125,
+      "rewards//std": 0.02214088849723339,
+      "step": 1784
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.357,
+      "grad_norm": 12.344344139099121,
+      "kl": 5.548849571496248,
+      "learning_rate": 7.265710289572328e-07,
+      "loss": 0.222,
+      "num_tokens": 15427997.0,
+      "reward": 0.77801513671875,
+      "reward_std": 0.016352981328964233,
+      "rewards//mean": 0.77801513671875,
+      "rewards//std": 0.03220841661095619,
+      "step": 1785
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3572,
+      "grad_norm": 10.783181190490723,
+      "kl": 1.9642281122505665,
+      "learning_rate": 7.262881009133241e-07,
+      "loss": 0.0786,
+      "num_tokens": 15436629.0,
+      "reward": 0.753173828125,
+      "reward_std": 0.0023689866065979004,
+      "rewards//mean": 0.753173828125,
+      "rewards//std": 0.018803080543875694,
+      "step": 1786
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3574,
+      "grad_norm": 5.746514797210693,
+      "kl": 3.0543887242674828,
+      "learning_rate": 7.260050817205955e-07,
+      "loss": 0.1222,
+      "num_tokens": 15445237.0,
+      "reward": 0.76434326171875,
+      "reward_std": 0.01005857065320015,
+      "rewards//mean": 0.76434326171875,
+      "rewards//std": 0.02969251573085785,
+      "step": 1787
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3576,
+      "grad_norm": 9.588447570800781,
+      "kl": 6.2093190886080265,
+      "learning_rate": 7.25721971493047e-07,
+      "loss": 0.2484,
+      "num_tokens": 15453805.0,
+      "reward": 0.75616455078125,
+      "reward_std": 0.010142859071493149,
+      "rewards//mean": 0.75616455078125,
+      "rewards//std": 0.028669537976384163,
+      "step": 1788
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3578,
+      "grad_norm": 9.859051704406738,
+      "kl": 5.229010626673698,
+      "learning_rate": 7.254387703447153e-07,
+      "loss": 0.2092,
+      "num_tokens": 15462397.0,
+      "reward": 0.76214599609375,
+      "reward_std": 0.01515856571495533,
+      "rewards//mean": 0.76214599609375,
+      "rewards//std": 0.030986279249191284,
+      "step": 1789
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.358,
+      "grad_norm": 8.220314025878906,
+      "kl": 4.179319351911545,
+      "learning_rate": 7.25155478389674e-07,
+      "loss": 0.1672,
+      "num_tokens": 15471029.0,
+      "reward": 0.76385498046875,
+      "reward_std": 0.01024538278579712,
+      "rewards//mean": 0.76385498046875,
+      "rewards//std": 0.024119237437844276,
+      "step": 1790
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3582,
+      "grad_norm": 4.266323089599609,
+      "kl": 4.320960188284516,
+      "learning_rate": 7.248720957420329e-07,
+      "loss": 0.1728,
+      "num_tokens": 15479709.0,
+      "reward": 0.77239990234375,
+      "reward_std": 0.010997525416314602,
+      "rewards//mean": 0.77239990234375,
+      "rewards//std": 0.026256943121552467,
+      "step": 1791
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3584,
+      "grad_norm": 7.368237018585205,
+      "kl": 2.9156788084656,
+      "learning_rate": 7.245886225159386e-07,
+      "loss": 0.1166,
+      "num_tokens": 15488325.0,
+      "reward": 0.717041015625,
+      "reward_std": 0.009482947178184986,
+      "rewards//mean": 0.717041015625,
+      "rewards//std": 0.045399196445941925,
+      "step": 1792
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3586,
+      "grad_norm": 14.691852569580078,
+      "kl": 1.7800093349069357,
+      "learning_rate": 7.243050588255737e-07,
+      "loss": 0.0712,
+      "num_tokens": 15496949.0,
+      "reward": 0.7899169921875,
+      "reward_std": 0.011507879011332989,
+      "rewards//mean": 0.7899169921875,
+      "rewards//std": 0.028069043532013893,
+      "step": 1793
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3588,
+      "grad_norm": 15.345220565795898,
+      "kl": 5.1895691603422165,
+      "learning_rate": 7.240214047851581e-07,
+      "loss": 0.2076,
+      "num_tokens": 15505589.0,
+      "reward": 0.7374267578125,
+      "reward_std": 0.0077485572546720505,
+      "rewards//mean": 0.7374267578125,
+      "rewards//std": 0.024394214153289795,
+      "step": 1794
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.359,
+      "grad_norm": 20.31637954711914,
+      "kl": 6.3183458019047976,
+      "learning_rate": 7.237376605089476e-07,
+      "loss": 0.2527,
+      "num_tokens": 15514253.0,
+      "reward": 0.776123046875,
+      "reward_std": 0.010714283213019371,
+      "rewards//mean": 0.776123046875,
+      "rewards//std": 0.025712909176945686,
+      "step": 1795
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3592,
+      "grad_norm": 7.15415620803833,
+      "kl": 4.081360448151827,
+      "learning_rate": 7.234538261112341e-07,
+      "loss": 0.1633,
+      "num_tokens": 15522845.0,
+      "reward": 0.7509765625,
+      "reward_std": 0.010711642913520336,
+      "rewards//mean": 0.7509765625,
+      "rewards//std": 0.02026761695742607,
+      "step": 1796
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3594,
+      "grad_norm": 5.400107383728027,
+      "kl": 3.705445107072592,
+      "learning_rate": 7.23169901706346e-07,
+      "loss": 0.1482,
+      "num_tokens": 15531453.0,
+      "reward": 0.74517822265625,
+      "reward_std": 0.009501457214355469,
+      "rewards//mean": 0.74517822265625,
+      "rewards//std": 0.03087371401488781,
+      "step": 1797
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3596,
+      "grad_norm": 10.734845161437988,
+      "kl": 5.157490076497197,
+      "learning_rate": 7.228858874086484e-07,
+      "loss": 0.2063,
+      "num_tokens": 15540037.0,
+      "reward": 0.7705078125,
+      "reward_std": 0.010244790464639664,
+      "rewards//mean": 0.7705078125,
+      "rewards//std": 0.01782611384987831,
+      "step": 1798
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3598,
+      "grad_norm": 10.73138427734375,
+      "kl": 2.7920976169407368,
+      "learning_rate": 7.226017833325419e-07,
+      "loss": 0.1117,
+      "num_tokens": 15548613.0,
+      "reward": 0.74395751953125,
+      "reward_std": 0.007455380167812109,
+      "rewards//mean": 0.74395751953125,
+      "rewards//std": 0.02261633798480034,
+      "step": 1799
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.36,
+      "grad_norm": 10.345914840698242,
+      "kl": 3.87006332911551,
+      "learning_rate": 7.223175895924637e-07,
+      "loss": 0.1548,
+      "num_tokens": 15557189.0,
+      "reward": 0.79193115234375,
+      "reward_std": 0.008266190066933632,
+      "rewards//mean": 0.79193115234375,
+      "rewards//std": 0.022000884637236595,
+      "step": 1800
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3602,
+      "grad_norm": 10.194170951843262,
+      "kl": 5.555856931954622,
+      "learning_rate": 7.220333063028871e-07,
+      "loss": 0.2222,
+      "num_tokens": 15565741.0,
+      "reward": 0.7454833984375,
+      "reward_std": 0.0249344352632761,
+      "rewards//mean": 0.7454833984375,
+      "rewards//std": 0.03213813528418541,
+      "step": 1801
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3604,
+      "grad_norm": 11.115693092346191,
+      "kl": 4.155972799286246,
+      "learning_rate": 7.217489335783211e-07,
+      "loss": 0.1662,
+      "num_tokens": 15574533.0,
+      "reward": 0.79888916015625,
+      "reward_std": 0.013462554663419724,
+      "rewards//mean": 0.79888916015625,
+      "rewards//std": 0.03218161314725876,
+      "step": 1802
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3606,
+      "grad_norm": 17.077611923217773,
+      "kl": 2.78132888302207,
+      "learning_rate": 7.214644715333114e-07,
+      "loss": 0.1113,
+      "num_tokens": 15583125.0,
+      "reward": 0.75701904296875,
+      "reward_std": 0.013729956932365894,
+      "rewards//mean": 0.75701904296875,
+      "rewards//std": 0.025884754955768585,
+      "step": 1803
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3608,
+      "grad_norm": 12.110036849975586,
+      "kl": 4.024367205798626,
+      "learning_rate": 7.211799202824388e-07,
+      "loss": 0.161,
+      "num_tokens": 15591757.0,
+      "reward": 0.74847412109375,
+      "reward_std": 0.011821866966784,
+      "rewards//mean": 0.74847412109375,
+      "rewards//std": 0.028208041563630104,
+      "step": 1804
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.361,
+      "grad_norm": 12.529596328735352,
+      "kl": 2.554512431845069,
+      "learning_rate": 7.20895279940321e-07,
+      "loss": 0.1022,
+      "num_tokens": 15600341.0,
+      "reward": 0.748046875,
+      "reward_std": 0.006430302746593952,
+      "rewards//mean": 0.748046875,
+      "rewards//std": 0.02427009865641594,
+      "step": 1805
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3612,
+      "grad_norm": 18.0447998046875,
+      "kl": 2.894049622118473,
+      "learning_rate": 7.206105506216106e-07,
+      "loss": 0.1158,
+      "num_tokens": 15608949.0,
+      "reward": 0.7611083984375,
+      "reward_std": 0.01057471800595522,
+      "rewards//mean": 0.7611083984375,
+      "rewards//std": 0.026182828471064568,
+      "step": 1806
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3614,
+      "grad_norm": 4.179669380187988,
+      "kl": 3.3387136291712523,
+      "learning_rate": 7.203257324409971e-07,
+      "loss": 0.1335,
+      "num_tokens": 15617549.0,
+      "reward": 0.76190185546875,
+      "reward_std": 0.009570148773491383,
+      "rewards//mean": 0.76190185546875,
+      "rewards//std": 0.03266298025846481,
+      "step": 1807
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3616,
+      "grad_norm": 7.232208251953125,
+      "kl": 3.486246306449175,
+      "learning_rate": 7.200408255132045e-07,
+      "loss": 0.1394,
+      "num_tokens": 15626093.0,
+      "reward": 0.7607421875,
+      "reward_std": 0.011610465124249458,
+      "rewards//mean": 0.7607421875,
+      "rewards//std": 0.02033919282257557,
+      "step": 1808
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3618,
+      "grad_norm": 16.169591903686523,
+      "kl": 1.3782273046672344,
+      "learning_rate": 7.19755829952994e-07,
+      "loss": 0.0551,
+      "num_tokens": 15634789.0,
+      "reward": 0.793212890625,
+      "reward_std": 0.006641528103500605,
+      "rewards//mean": 0.793212890625,
+      "rewards//std": 0.015975650399923325,
+      "step": 1809
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.362,
+      "grad_norm": 12.412141799926758,
+      "kl": 3.3542172610759735,
+      "learning_rate": 7.194707458751615e-07,
+      "loss": 0.1342,
+      "num_tokens": 15643493.0,
+      "reward": 0.7469482421875,
+      "reward_std": 0.007622177712619305,
+      "rewards//mean": 0.7469482421875,
+      "rewards//std": 0.027982624247670174,
+      "step": 1810
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3622,
+      "grad_norm": 11.963812828063965,
+      "kl": 4.771061383187771,
+      "learning_rate": 7.191855733945386e-07,
+      "loss": 0.1908,
+      "num_tokens": 15652381.0,
+      "reward": 0.7396240234375,
+      "reward_std": 0.01201414130628109,
+      "rewards//mean": 0.7396240234375,
+      "rewards//std": 0.03593115508556366,
+      "step": 1811
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3624,
+      "grad_norm": 10.7211332321167,
+      "kl": 1.8340128250420094,
+      "learning_rate": 7.189003126259931e-07,
+      "loss": 0.0734,
+      "num_tokens": 15661125.0,
+      "reward": 0.7752685546875,
+      "reward_std": 0.0092721376568079,
+      "rewards//mean": 0.7752685546875,
+      "rewards//std": 0.02249864861369133,
+      "step": 1812
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3626,
+      "grad_norm": 14.77794361114502,
+      "kl": 1.1398207396268845,
+      "learning_rate": 7.186149636844279e-07,
+      "loss": 0.0456,
+      "num_tokens": 15669845.0,
+      "reward": 0.778076171875,
+      "reward_std": 0.003906923346221447,
+      "rewards//mean": 0.778076171875,
+      "rewards//std": 0.015309864655137062,
+      "step": 1813
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3628,
+      "grad_norm": 17.214759826660156,
+      "kl": 2.4941138923168182,
+      "learning_rate": 7.183295266847814e-07,
+      "loss": 0.0998,
+      "num_tokens": 15678469.0,
+      "reward": 0.7567138671875,
+      "reward_std": 0.007382056210190058,
+      "rewards//mean": 0.7567138671875,
+      "rewards//std": 0.02023734524846077,
+      "step": 1814
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.363,
+      "grad_norm": 8.335517883300781,
+      "kl": 2.8564509116113186,
+      "learning_rate": 7.180440017420276e-07,
+      "loss": 0.1143,
+      "num_tokens": 15687037.0,
+      "reward": 0.747802734375,
+      "reward_std": 0.005393965169787407,
+      "rewards//mean": 0.747802734375,
+      "rewards//std": 0.027320489287376404,
+      "step": 1815
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3632,
+      "grad_norm": 8.306028366088867,
+      "kl": 3.8441889118403196,
+      "learning_rate": 7.177583889711762e-07,
+      "loss": 0.1538,
+      "num_tokens": 15695685.0,
+      "reward": 0.77984619140625,
+      "reward_std": 0.00937097892165184,
+      "rewards//mean": 0.77984619140625,
+      "rewards//std": 0.025385782122612,
+      "step": 1816
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3634,
+      "grad_norm": 15.702668190002441,
+      "kl": 5.227973237633705,
+      "learning_rate": 7.174726884872715e-07,
+      "loss": 0.2091,
+      "num_tokens": 15704253.0,
+      "reward": 0.7515869140625,
+      "reward_std": 0.009687444195151329,
+      "rewards//mean": 0.7515869140625,
+      "rewards//std": 0.029876815155148506,
+      "step": 1817
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3636,
+      "grad_norm": 18.41916275024414,
+      "kl": 6.280253957957029,
+      "learning_rate": 7.17186900405394e-07,
+      "loss": 0.2512,
+      "num_tokens": 15712845.0,
+      "reward": 0.74658203125,
+      "reward_std": 0.017452437430620193,
+      "rewards//mean": 0.74658203125,
+      "rewards//std": 0.0323505662381649,
+      "step": 1818
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3638,
+      "grad_norm": 28.84602165222168,
+      "kl": 7.063274849206209,
+      "learning_rate": 7.169010248406588e-07,
+      "loss": 0.2825,
+      "num_tokens": 15721461.0,
+      "reward": 0.75006103515625,
+      "reward_std": 0.011782647110521793,
+      "rewards//mean": 0.75006103515625,
+      "rewards//std": 0.028768105432391167,
+      "step": 1819
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.364,
+      "grad_norm": 6.0525007247924805,
+      "kl": 3.8506972566246986,
+      "learning_rate": 7.16615061908217e-07,
+      "loss": 0.154,
+      "num_tokens": 15730149.0,
+      "reward": 0.7410888671875,
+      "reward_std": 0.007551171816885471,
+      "rewards//mean": 0.7410888671875,
+      "rewards//std": 0.03739281743764877,
+      "step": 1820
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3642,
+      "grad_norm": 7.779174327850342,
+      "kl": 2.575003158301115,
+      "learning_rate": 7.163290117232541e-07,
+      "loss": 0.103,
+      "num_tokens": 15738717.0,
+      "reward": 0.75946044921875,
+      "reward_std": 0.006207785569131374,
+      "rewards//mean": 0.75946044921875,
+      "rewards//std": 0.020085208117961884,
+      "step": 1821
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3644,
+      "grad_norm": 2.5102193355560303,
+      "kl": 1.606106111779809,
+      "learning_rate": 7.160428744009912e-07,
+      "loss": 0.0642,
+      "num_tokens": 15747325.0,
+      "reward": 0.73175048828125,
+      "reward_std": 0.004206686746329069,
+      "rewards//mean": 0.73175048828125,
+      "rewards//std": 0.022359192371368408,
+      "step": 1822
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3646,
+      "grad_norm": 7.623538017272949,
+      "kl": 4.200133021920919,
+      "learning_rate": 7.157566500566842e-07,
+      "loss": 0.168,
+      "num_tokens": 15755973.0,
+      "reward": 0.78436279296875,
+      "reward_std": 0.011930635198950768,
+      "rewards//mean": 0.78436279296875,
+      "rewards//std": 0.023490967229008675,
+      "step": 1823
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3648,
+      "grad_norm": 6.204391002655029,
+      "kl": 2.8439120296388865,
+      "learning_rate": 7.154703388056244e-07,
+      "loss": 0.1138,
+      "num_tokens": 15764549.0,
+      "reward": 0.76434326171875,
+      "reward_std": 0.008812109008431435,
+      "rewards//mean": 0.76434326171875,
+      "rewards//std": 0.01907394640147686,
+      "step": 1824
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.365,
+      "grad_norm": 11.921187400817871,
+      "kl": 1.9189935978502035,
+      "learning_rate": 7.15183940763138e-07,
+      "loss": 0.0768,
+      "num_tokens": 15773165.0,
+      "reward": 0.75701904296875,
+      "reward_std": 0.005657212808728218,
+      "rewards//mean": 0.75701904296875,
+      "rewards//std": 0.0242119450122118,
+      "step": 1825
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3652,
+      "grad_norm": 12.116460800170898,
+      "kl": 2.1174923833459616,
+      "learning_rate": 7.148974560445858e-07,
+      "loss": 0.0847,
+      "num_tokens": 15781861.0,
+      "reward": 0.822265625,
+      "reward_std": 0.011779746040701866,
+      "rewards//mean": 0.822265625,
+      "rewards//std": 0.023540614172816277,
+      "step": 1826
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3654,
+      "grad_norm": 20.339641571044922,
+      "kl": 6.685268387198448,
+      "learning_rate": 7.146108847653641e-07,
+      "loss": 0.2674,
+      "num_tokens": 15790461.0,
+      "reward": 0.74822998046875,
+      "reward_std": 0.010201862081885338,
+      "rewards//mean": 0.74822998046875,
+      "rewards//std": 0.0224874597042799,
+      "step": 1827
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3656,
+      "grad_norm": 4.9110612869262695,
+      "kl": 2.461530778557062,
+      "learning_rate": 7.143242270409037e-07,
+      "loss": 0.0985,
+      "num_tokens": 15799221.0,
+      "reward": 0.75177001953125,
+      "reward_std": 0.004146665334701538,
+      "rewards//mean": 0.75177001953125,
+      "rewards//std": 0.030233096331357956,
+      "step": 1828
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3658,
+      "grad_norm": 5.22251033782959,
+      "kl": 1.3586306106299162,
+      "learning_rate": 7.140374829866702e-07,
+      "loss": 0.0543,
+      "num_tokens": 15807805.0,
+      "reward": 0.748046875,
+      "reward_std": 0.00377071974799037,
+      "rewards//mean": 0.748046875,
+      "rewards//std": 0.018727252259850502,
+      "step": 1829
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.366,
+      "grad_norm": 4.655337810516357,
+      "kl": 4.365455897524953,
+      "learning_rate": 7.137506527181643e-07,
+      "loss": 0.1746,
+      "num_tokens": 15816461.0,
+      "reward": 0.74749755859375,
+      "reward_std": 0.010998677462339401,
+      "rewards//mean": 0.74749755859375,
+      "rewards//std": 0.029333990067243576,
+      "step": 1830
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3662,
+      "grad_norm": 11.291488647460938,
+      "kl": 4.101958701387048,
+      "learning_rate": 7.134637363509209e-07,
+      "loss": 0.1641,
+      "num_tokens": 15825077.0,
+      "reward": 0.7510986328125,
+      "reward_std": 0.00990404561161995,
+      "rewards//mean": 0.7510986328125,
+      "rewards//std": 0.026887930929660797,
+      "step": 1831
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3664,
+      "grad_norm": 4.310554027557373,
+      "kl": 1.6261647623032331,
+      "learning_rate": 7.131767340005101e-07,
+      "loss": 0.065,
+      "num_tokens": 15833709.0,
+      "reward": 0.7259521484375,
+      "reward_std": 0.004137130919843912,
+      "rewards//mean": 0.7259521484375,
+      "rewards//std": 0.025546055287122726,
+      "step": 1832
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3666,
+      "grad_norm": 6.229635238647461,
+      "kl": 3.4514841958880424,
+      "learning_rate": 7.128896457825363e-07,
+      "loss": 0.1381,
+      "num_tokens": 15842405.0,
+      "reward": 0.7718505859375,
+      "reward_std": 0.00895756483078003,
+      "rewards//mean": 0.7718505859375,
+      "rewards//std": 0.026399319991469383,
+      "step": 1833
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3668,
+      "grad_norm": 11.113919258117676,
+      "kl": 3.233485661447048,
+      "learning_rate": 7.126024718126387e-07,
+      "loss": 0.1293,
+      "num_tokens": 15851117.0,
+      "reward": 0.7513427734375,
+      "reward_std": 0.009316753596067429,
+      "rewards//mean": 0.7513427734375,
+      "rewards//std": 0.024931885302066803,
+      "step": 1834
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.367,
+      "grad_norm": 6.431647300720215,
+      "kl": 3.877834590151906,
+      "learning_rate": 7.123152122064908e-07,
+      "loss": 0.1551,
+      "num_tokens": 15859685.0,
+      "reward": 0.7598876953125,
+      "reward_std": 0.008115262724459171,
+      "rewards//mean": 0.7598876953125,
+      "rewards//std": 0.02669356018304825,
+      "step": 1835
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3672,
+      "grad_norm": 5.051802635192871,
+      "kl": 3.4883563816547394,
+      "learning_rate": 7.120278670798009e-07,
+      "loss": 0.1395,
+      "num_tokens": 15868293.0,
+      "reward": 0.7637939453125,
+      "reward_std": 0.009110360406339169,
+      "rewards//mean": 0.7637939453125,
+      "rewards//std": 0.031970009207725525,
+      "step": 1836
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 126.5625,
+      "epoch": 0.3674,
+      "grad_norm": 3.741506576538086,
+      "kl": 2.80803806707263,
+      "learning_rate": 7.117404365483115e-07,
+      "loss": 0.1287,
+      "num_tokens": 15876961.0,
+      "reward": 0.79388427734375,
+      "reward_std": 0.0073487600311636925,
+      "rewards//mean": 0.79388427734375,
+      "rewards//std": 0.02423756755888462,
+      "step": 1837
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3676,
+      "grad_norm": 8.29547119140625,
+      "kl": 2.961475620046258,
+      "learning_rate": 7.114529207277995e-07,
+      "loss": 0.1185,
+      "num_tokens": 15885633.0,
+      "reward": 0.76275634765625,
+      "reward_std": 0.008384748362004757,
+      "rewards//mean": 0.76275634765625,
+      "rewards//std": 0.0255747027695179,
+      "step": 1838
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3678,
+      "grad_norm": 3.3730926513671875,
+      "kl": 3.096415415406227,
+      "learning_rate": 7.111653197340764e-07,
+      "loss": 0.1239,
+      "num_tokens": 15894241.0,
+      "reward": 0.73614501953125,
+      "reward_std": 0.0058305514976382256,
+      "rewards//mean": 0.73614501953125,
+      "rewards//std": 0.031473420560359955,
+      "step": 1839
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.368,
+      "grad_norm": 5.94202184677124,
+      "kl": 2.1518924087285995,
+      "learning_rate": 7.108776336829876e-07,
+      "loss": 0.0861,
+      "num_tokens": 15902793.0,
+      "reward": 0.76837158203125,
+      "reward_std": 0.0043298485688865185,
+      "rewards//mean": 0.76837158203125,
+      "rewards//std": 0.017916640266776085,
+      "step": 1840
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3682,
+      "grad_norm": 14.551260948181152,
+      "kl": 4.650878993794322,
+      "learning_rate": 7.105898626904134e-07,
+      "loss": 0.186,
+      "num_tokens": 15911425.0,
+      "reward": 0.797119140625,
+      "reward_std": 0.009744038805365562,
+      "rewards//mean": 0.797119140625,
+      "rewards//std": 0.0221941526979208,
+      "step": 1841
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3684,
+      "grad_norm": 3.400298833847046,
+      "kl": 1.1997675318270922,
+      "learning_rate": 7.103020068722674e-07,
+      "loss": 0.048,
+      "num_tokens": 15920129.0,
+      "reward": 0.7310791015625,
+      "reward_std": 0.00397300161421299,
+      "rewards//mean": 0.7310791015625,
+      "rewards//std": 0.028196033090353012,
+      "step": 1842
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3686,
+      "grad_norm": 5.189095497131348,
+      "kl": 2.531966695562005,
+      "learning_rate": 7.100140663444984e-07,
+      "loss": 0.1013,
+      "num_tokens": 15928817.0,
+      "reward": 0.7432861328125,
+      "reward_std": 0.005302074830979109,
+      "rewards//mean": 0.7432861328125,
+      "rewards//std": 0.03379307687282562,
+      "step": 1843
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3688,
+      "grad_norm": 5.2451958656311035,
+      "kl": 0.819163890555501,
+      "learning_rate": 7.097260412230885e-07,
+      "loss": 0.0328,
+      "num_tokens": 15937457.0,
+      "reward": 0.7691650390625,
+      "reward_std": 0.0031269120518118143,
+      "rewards//mean": 0.7691650390625,
+      "rewards//std": 0.02102680318057537,
+      "step": 1844
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.369,
+      "grad_norm": 1.9537146091461182,
+      "kl": 2.4095728807151318,
+      "learning_rate": 7.094379316240544e-07,
+      "loss": 0.0964,
+      "num_tokens": 15946089.0,
+      "reward": 0.7491455078125,
+      "reward_std": 0.007642756681889296,
+      "rewards//mean": 0.7491455078125,
+      "rewards//std": 0.028960904106497765,
+      "step": 1845
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3692,
+      "grad_norm": 9.979609489440918,
+      "kl": 1.6186884567141533,
+      "learning_rate": 7.091497376634463e-07,
+      "loss": 0.0647,
+      "num_tokens": 15954761.0,
+      "reward": 0.7952880859375,
+      "reward_std": 0.005419217981398106,
+      "rewards//mean": 0.7952880859375,
+      "rewards//std": 0.023681342601776123,
+      "step": 1846
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3694,
+      "grad_norm": 12.618873596191406,
+      "kl": 3.9344957154244184,
+      "learning_rate": 7.088614594573491e-07,
+      "loss": 0.1574,
+      "num_tokens": 15963369.0,
+      "reward": 0.76629638671875,
+      "reward_std": 0.01030784659087658,
+      "rewards//mean": 0.76629638671875,
+      "rewards//std": 0.029120083898305893,
+      "step": 1847
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3696,
+      "grad_norm": 9.172502517700195,
+      "kl": 2.6048205718398094,
+      "learning_rate": 7.085730971218809e-07,
+      "loss": 0.1042,
+      "num_tokens": 15971985.0,
+      "reward": 0.77740478515625,
+      "reward_std": 0.006511168088763952,
+      "rewards//mean": 0.77740478515625,
+      "rewards//std": 0.019960464909672737,
+      "step": 1848
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3698,
+      "grad_norm": 7.123248100280762,
+      "kl": 1.7873948123306036,
+      "learning_rate": 7.082846507731941e-07,
+      "loss": 0.0715,
+      "num_tokens": 15980625.0,
+      "reward": 0.73388671875,
+      "reward_std": 0.005009832326322794,
+      "rewards//mean": 0.73388671875,
+      "rewards//std": 0.022771839052438736,
+      "step": 1849
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.37,
+      "grad_norm": 14.527019500732422,
+      "kl": 3.1183022148907185,
+      "learning_rate": 7.079961205274748e-07,
+      "loss": 0.1247,
+      "num_tokens": 15989297.0,
+      "reward": 0.78668212890625,
+      "reward_std": 0.006082155276089907,
+      "rewards//mean": 0.78668212890625,
+      "rewards//std": 0.017310744151473045,
+      "step": 1850
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3702,
+      "grad_norm": 6.804238796234131,
+      "kl": 4.943616297096014,
+      "learning_rate": 7.077075065009433e-07,
+      "loss": 0.1977,
+      "num_tokens": 15997929.0,
+      "reward": 0.79132080078125,
+      "reward_std": 0.012570510618388653,
+      "rewards//mean": 0.79132080078125,
+      "rewards//std": 0.026836862787604332,
+      "step": 1851
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3704,
+      "grad_norm": 5.623232364654541,
+      "kl": 1.2311439421027899,
+      "learning_rate": 7.074188088098527e-07,
+      "loss": 0.0492,
+      "num_tokens": 16006505.0,
+      "reward": 0.78045654296875,
+      "reward_std": 0.005158598534762859,
+      "rewards//mean": 0.78045654296875,
+      "rewards//std": 0.025297973304986954,
+      "step": 1852
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3706,
+      "grad_norm": 3.320368766784668,
+      "kl": 3.7772607430815697,
+      "learning_rate": 7.071300275704909e-07,
+      "loss": 0.1511,
+      "num_tokens": 16015257.0,
+      "reward": 0.7457275390625,
+      "reward_std": 0.013418788090348244,
+      "rewards//mean": 0.7457275390625,
+      "rewards//std": 0.02494645304977894,
+      "step": 1853
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3708,
+      "grad_norm": 7.783235549926758,
+      "kl": 3.596936283633113,
+      "learning_rate": 7.068411628991787e-07,
+      "loss": 0.1439,
+      "num_tokens": 16023985.0,
+      "reward": 0.77288818359375,
+      "reward_std": 0.0066768997348845005,
+      "rewards//mean": 0.77288818359375,
+      "rewards//std": 0.043218113481998444,
+      "step": 1854
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.371,
+      "grad_norm": 5.301913738250732,
+      "kl": 0.6076066605746746,
+      "learning_rate": 7.065522149122709e-07,
+      "loss": 0.0243,
+      "num_tokens": 16032585.0,
+      "reward": 0.73272705078125,
+      "reward_std": 0.0008328249678015709,
+      "rewards//mean": 0.73272705078125,
+      "rewards//std": 0.022518403828144073,
+      "step": 1855
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3712,
+      "grad_norm": 2.3196868896484375,
+      "kl": 0.8560703378170729,
+      "learning_rate": 7.062631837261556e-07,
+      "loss": 0.0342,
+      "num_tokens": 16041097.0,
+      "reward": 0.79913330078125,
+      "reward_std": 0.002766229910776019,
+      "rewards//mean": 0.79913330078125,
+      "rewards//std": 0.016874125227332115,
+      "step": 1856
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3714,
+      "grad_norm": 3.6160199642181396,
+      "kl": 2.6725729685276747,
+      "learning_rate": 7.059740694572545e-07,
+      "loss": 0.1069,
+      "num_tokens": 16049721.0,
+      "reward": 0.74658203125,
+      "reward_std": 0.005142111796885729,
+      "rewards//mean": 0.74658203125,
+      "rewards//std": 0.020640617236495018,
+      "step": 1857
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3716,
+      "grad_norm": 5.1055006980896,
+      "kl": 3.198473073542118,
+      "learning_rate": 7.056848722220228e-07,
+      "loss": 0.1279,
+      "num_tokens": 16058401.0,
+      "reward": 0.7724609375,
+      "reward_std": 0.011910544708371162,
+      "rewards//mean": 0.7724609375,
+      "rewards//std": 0.023765888065099716,
+      "step": 1858
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3718,
+      "grad_norm": 11.609858512878418,
+      "kl": 1.2559426296502352,
+      "learning_rate": 7.053955921369493e-07,
+      "loss": 0.0502,
+      "num_tokens": 16067001.0,
+      "reward": 0.77606201171875,
+      "reward_std": 0.008136093616485596,
+      "rewards//mean": 0.77606201171875,
+      "rewards//std": 0.019930865615606308,
+      "step": 1859
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.372,
+      "grad_norm": 4.9913811683654785,
+      "kl": 3.288194267079234,
+      "learning_rate": 7.051062293185559e-07,
+      "loss": 0.1315,
+      "num_tokens": 16075641.0,
+      "reward": 0.75323486328125,
+      "reward_std": 0.008526476100087166,
+      "rewards//mean": 0.75323486328125,
+      "rewards//std": 0.026789437979459763,
+      "step": 1860
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3722,
+      "grad_norm": 4.668308258056641,
+      "kl": 0.9770767856389284,
+      "learning_rate": 7.048167838833976e-07,
+      "loss": 0.0391,
+      "num_tokens": 16084201.0,
+      "reward": 0.75799560546875,
+      "reward_std": 0.0024019088596105576,
+      "rewards//mean": 0.75799560546875,
+      "rewards//std": 0.02114981971681118,
+      "step": 1861
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3724,
+      "grad_norm": 7.400339603424072,
+      "kl": 2.7208936866372824,
+      "learning_rate": 7.045272559480635e-07,
+      "loss": 0.1088,
+      "num_tokens": 16092793.0,
+      "reward": 0.75408935546875,
+      "reward_std": 0.007091997656971216,
+      "rewards//mean": 0.75408935546875,
+      "rewards//std": 0.02746685966849327,
+      "step": 1862
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3726,
+      "grad_norm": 5.857992172241211,
+      "kl": 2.533273993059993,
+      "learning_rate": 7.042376456291751e-07,
+      "loss": 0.1013,
+      "num_tokens": 16101465.0,
+      "reward": 0.735595703125,
+      "reward_std": 0.00920802727341652,
+      "rewards//mean": 0.735595703125,
+      "rewards//std": 0.03148069605231285,
+      "step": 1863
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3728,
+      "grad_norm": 13.461249351501465,
+      "kl": 3.350059609860182,
+      "learning_rate": 7.039479530433874e-07,
+      "loss": 0.134,
+      "num_tokens": 16110065.0,
+      "reward": 0.77325439453125,
+      "reward_std": 0.001975295366719365,
+      "rewards//mean": 0.77325439453125,
+      "rewards//std": 0.022079868242144585,
+      "step": 1864
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.373,
+      "grad_norm": 25.732999801635742,
+      "kl": 0.7749489434063435,
+      "learning_rate": 7.036581783073887e-07,
+      "loss": 0.031,
+      "num_tokens": 16118681.0,
+      "reward": 0.74041748046875,
+      "reward_std": 0.0021365329157561064,
+      "rewards//mean": 0.74041748046875,
+      "rewards//std": 0.020433401688933372,
+      "step": 1865
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3732,
+      "grad_norm": 4.1407470703125,
+      "kl": 2.4588584788143635,
+      "learning_rate": 7.033683215379002e-07,
+      "loss": 0.0984,
+      "num_tokens": 16127433.0,
+      "reward": 0.77386474609375,
+      "reward_std": 0.004057341255247593,
+      "rewards//mean": 0.77386474609375,
+      "rewards//std": 0.0292575154453516,
+      "step": 1866
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3734,
+      "grad_norm": 10.789741516113281,
+      "kl": 2.492692956700921,
+      "learning_rate": 7.030783828516759e-07,
+      "loss": 0.0997,
+      "num_tokens": 16136129.0,
+      "reward": 0.75921630859375,
+      "reward_std": 0.0030138352885842323,
+      "rewards//mean": 0.75921630859375,
+      "rewards//std": 0.020414873957633972,
+      "step": 1867
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3736,
+      "grad_norm": 4.771057605743408,
+      "kl": 1.283412892371416,
+      "learning_rate": 7.027883623655034e-07,
+      "loss": 0.0513,
+      "num_tokens": 16144681.0,
+      "reward": 0.7633056640625,
+      "reward_std": 0.002828120719641447,
+      "rewards//mean": 0.7633056640625,
+      "rewards//std": 0.01930948905646801,
+      "step": 1868
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3738,
+      "grad_norm": 6.336138725280762,
+      "kl": 2.7801638580858707,
+      "learning_rate": 7.024982601962026e-07,
+      "loss": 0.1112,
+      "num_tokens": 16153377.0,
+      "reward": 0.77020263671875,
+      "reward_std": 0.012073297053575516,
+      "rewards//mean": 0.77020263671875,
+      "rewards//std": 0.025972329080104828,
+      "step": 1869
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.374,
+      "grad_norm": 4.534235954284668,
+      "kl": 2.658373724669218,
+      "learning_rate": 7.022080764606271e-07,
+      "loss": 0.1063,
+      "num_tokens": 16161937.0,
+      "reward": 0.73101806640625,
+      "reward_std": 0.006083108484745026,
+      "rewards//mean": 0.73101806640625,
+      "rewards//std": 0.03175640106201172,
+      "step": 1870
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3742,
+      "grad_norm": 2.8037991523742676,
+      "kl": 1.569754533469677,
+      "learning_rate": 7.019178112756625e-07,
+      "loss": 0.0628,
+      "num_tokens": 16170625.0,
+      "reward": 0.764404296875,
+      "reward_std": 0.005482660606503487,
+      "rewards//mean": 0.764404296875,
+      "rewards//std": 0.028038017451763153,
+      "step": 1871
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3744,
+      "grad_norm": 2.921419382095337,
+      "kl": 1.2974753342568874,
+      "learning_rate": 7.016274647582276e-07,
+      "loss": 0.0519,
+      "num_tokens": 16179305.0,
+      "reward": 0.779296875,
+      "reward_std": 0.006271827034652233,
+      "rewards//mean": 0.779296875,
+      "rewards//std": 0.020505229011178017,
+      "step": 1872
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3746,
+      "grad_norm": 7.523586273193359,
+      "kl": 1.884477224200964,
+      "learning_rate": 7.013370370252739e-07,
+      "loss": 0.0754,
+      "num_tokens": 16187841.0,
+      "reward": 0.7667236328125,
+      "reward_std": 0.0027007656171917915,
+      "rewards//mean": 0.7667236328125,
+      "rewards//std": 0.011676693335175514,
+      "step": 1873
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3748,
+      "grad_norm": 1.177193284034729,
+      "kl": 0.48449935764074326,
+      "learning_rate": 7.010465281937858e-07,
+      "loss": 0.0194,
+      "num_tokens": 16196409.0,
+      "reward": 0.80242919921875,
+      "reward_std": 0.0006246941047720611,
+      "rewards//mean": 0.80242919921875,
+      "rewards//std": 0.009664148092269897,
+      "step": 1874
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.375,
+      "grad_norm": 2.5623443126678467,
+      "kl": 0.8957772143185139,
+      "learning_rate": 7.007559383807802e-07,
+      "loss": 0.0358,
+      "num_tokens": 16205089.0,
+      "reward": 0.765380859375,
+      "reward_std": 0.0035510831512510777,
+      "rewards//mean": 0.765380859375,
+      "rewards//std": 0.021291865035891533,
+      "step": 1875
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3752,
+      "grad_norm": 2.6249661445617676,
+      "kl": 1.313623446971178,
+      "learning_rate": 7.004652677033068e-07,
+      "loss": 0.0525,
+      "num_tokens": 16213697.0,
+      "reward": 0.77490234375,
+      "reward_std": 0.0047293840907514095,
+      "rewards//mean": 0.77490234375,
+      "rewards//std": 0.01691276952624321,
+      "step": 1876
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3754,
+      "grad_norm": 6.697422504425049,
+      "kl": 2.700675642117858,
+      "learning_rate": 7.001745162784475e-07,
+      "loss": 0.108,
+      "num_tokens": 16222273.0,
+      "reward": 0.7401123046875,
+      "reward_std": 0.004058271646499634,
+      "rewards//mean": 0.7401123046875,
+      "rewards//std": 0.022052858024835587,
+      "step": 1877
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3756,
+      "grad_norm": 2.777811288833618,
+      "kl": 1.570582689717412,
+      "learning_rate": 6.998836842233169e-07,
+      "loss": 0.0628,
+      "num_tokens": 16230945.0,
+      "reward": 0.7818603515625,
+      "reward_std": 0.004180000629276037,
+      "rewards//mean": 0.7818603515625,
+      "rewards//std": 0.020997988060116768,
+      "step": 1878
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3758,
+      "grad_norm": 4.907735824584961,
+      "kl": 2.7225206568837166,
+      "learning_rate": 6.995927716550622e-07,
+      "loss": 0.1089,
+      "num_tokens": 16239593.0,
+      "reward": 0.773681640625,
+      "reward_std": 0.006213858723640442,
+      "rewards//mean": 0.773681640625,
+      "rewards//std": 0.02637321501970291,
+      "step": 1879
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.376,
+      "grad_norm": 2.8154067993164062,
+      "kl": 1.4785632863640785,
+      "learning_rate": 6.99301778690863e-07,
+      "loss": 0.0591,
+      "num_tokens": 16248241.0,
+      "reward": 0.76678466796875,
+      "reward_std": 0.008459039963781834,
+      "rewards//mean": 0.76678466796875,
+      "rewards//std": 0.02695280872285366,
+      "step": 1880
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3762,
+      "grad_norm": 0.7601634860038757,
+      "kl": 0.5685290694236755,
+      "learning_rate": 6.990107054479312e-07,
+      "loss": 0.0227,
+      "num_tokens": 16256897.0,
+      "reward": 0.76593017578125,
+      "reward_std": 0.00039866380393505096,
+      "rewards//mean": 0.76593017578125,
+      "rewards//std": 0.019578374922275543,
+      "step": 1881
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3764,
+      "grad_norm": 1.846854329109192,
+      "kl": 2.0518183298408985,
+      "learning_rate": 6.987195520435109e-07,
+      "loss": 0.0821,
+      "num_tokens": 16265569.0,
+      "reward": 0.73638916015625,
+      "reward_std": 0.004954901058226824,
+      "rewards//mean": 0.73638916015625,
+      "rewards//std": 0.025382203981280327,
+      "step": 1882
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3766,
+      "grad_norm": 3.0427513122558594,
+      "kl": 1.4905531387776136,
+      "learning_rate": 6.984283185948789e-07,
+      "loss": 0.0596,
+      "num_tokens": 16274201.0,
+      "reward": 0.77337646484375,
+      "reward_std": 0.0029347692616283894,
+      "rewards//mean": 0.77337646484375,
+      "rewards//std": 0.02367902360856533,
+      "step": 1883
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3768,
+      "grad_norm": 4.615382194519043,
+      "kl": 3.5652448013424873,
+      "learning_rate": 6.981370052193439e-07,
+      "loss": 0.1426,
+      "num_tokens": 16283041.0,
+      "reward": 0.7841796875,
+      "reward_std": 0.0068917106837034225,
+      "rewards//mean": 0.7841796875,
+      "rewards//std": 0.029190242290496826,
+      "step": 1884
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.377,
+      "grad_norm": 0.02688196487724781,
+      "kl": 0.5113901514559984,
+      "learning_rate": 6.978456120342469e-07,
+      "loss": 0.0205,
+      "num_tokens": 16291745.0,
+      "reward": 0.77978515625,
+      "reward_std": 0.0,
+      "rewards//mean": 0.77978515625,
+      "rewards//std": 0.017267076298594475,
+      "step": 1885
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3772,
+      "grad_norm": 1.5688022375106812,
+      "kl": 0.9871741626411676,
+      "learning_rate": 6.975541391569609e-07,
+      "loss": 0.0395,
+      "num_tokens": 16300337.0,
+      "reward": 0.76629638671875,
+      "reward_std": 0.00266617932356894,
+      "rewards//mean": 0.76629638671875,
+      "rewards//std": 0.02031005173921585,
+      "step": 1886
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3774,
+      "grad_norm": 6.302677154541016,
+      "kl": 3.641746509820223,
+      "learning_rate": 6.972625867048914e-07,
+      "loss": 0.1457,
+      "num_tokens": 16308905.0,
+      "reward": 0.7529296875,
+      "reward_std": 0.008294638246297836,
+      "rewards//mean": 0.7529296875,
+      "rewards//std": 0.032053474336862564,
+      "step": 1887
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3776,
+      "grad_norm": 1.8699313402175903,
+      "kl": 0.9070111233741045,
+      "learning_rate": 6.969709547954755e-07,
+      "loss": 0.0363,
+      "num_tokens": 16317513.0,
+      "reward": 0.7874755859375,
+      "reward_std": 0.0033482497092336416,
+      "rewards//mean": 0.7874755859375,
+      "rewards//std": 0.024416543543338776,
+      "step": 1888
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3778,
+      "grad_norm": 1.1291965246200562,
+      "kl": 1.054303988814354,
+      "learning_rate": 6.966792435461826e-07,
+      "loss": 0.0422,
+      "num_tokens": 16326153.0,
+      "reward": 0.78106689453125,
+      "reward_std": 0.003481745719909668,
+      "rewards//mean": 0.78106689453125,
+      "rewards//std": 0.023379866033792496,
+      "step": 1889
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.378,
+      "grad_norm": 4.352121829986572,
+      "kl": 0.525041151791811,
+      "learning_rate": 6.963874530745139e-07,
+      "loss": 0.021,
+      "num_tokens": 16334721.0,
+      "reward": 0.74969482421875,
+      "reward_std": 0.001633777399547398,
+      "rewards//mean": 0.74969482421875,
+      "rewards//std": 0.01619293913245201,
+      "step": 1890
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3782,
+      "grad_norm": 2.1457080841064453,
+      "kl": 0.546690184623003,
+      "learning_rate": 6.960955834980027e-07,
+      "loss": 0.0219,
+      "num_tokens": 16343401.0,
+      "reward": 0.74993896484375,
+      "reward_std": 0.0012657219776883721,
+      "rewards//mean": 0.74993896484375,
+      "rewards//std": 0.03805427625775337,
+      "step": 1891
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3784,
+      "grad_norm": 2.121290683746338,
+      "kl": 2.8007206730544567,
+      "learning_rate": 6.958036349342139e-07,
+      "loss": 0.112,
+      "num_tokens": 16351961.0,
+      "reward": 0.7564697265625,
+      "reward_std": 0.0069049508310854435,
+      "rewards//mean": 0.7564697265625,
+      "rewards//std": 0.026071587577462196,
+      "step": 1892
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3786,
+      "grad_norm": 1.729514479637146,
+      "kl": 2.4328966289758682,
+      "learning_rate": 6.955116075007442e-07,
+      "loss": 0.0973,
+      "num_tokens": 16360601.0,
+      "reward": 0.76934814453125,
+      "reward_std": 0.007147075608372688,
+      "rewards//mean": 0.76934814453125,
+      "rewards//std": 0.026119951158761978,
+      "step": 1893
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3788,
+      "grad_norm": 8.549952507019043,
+      "kl": 5.081495724618435,
+      "learning_rate": 6.952195013152225e-07,
+      "loss": 0.2033,
+      "num_tokens": 16369193.0,
+      "reward": 0.7581787109375,
+      "reward_std": 0.007141421549022198,
+      "rewards//mean": 0.7581787109375,
+      "rewards//std": 0.02632582001388073,
+      "step": 1894
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.379,
+      "grad_norm": 0.9424338340759277,
+      "kl": 1.179397590458393,
+      "learning_rate": 6.94927316495309e-07,
+      "loss": 0.0472,
+      "num_tokens": 16377833.0,
+      "reward": 0.7740478515625,
+      "reward_std": 0.0033621666952967644,
+      "rewards//mean": 0.7740478515625,
+      "rewards//std": 0.014947145245969296,
+      "step": 1895
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3792,
+      "grad_norm": 1.572584867477417,
+      "kl": 1.4091698415577412,
+      "learning_rate": 6.946350531586957e-07,
+      "loss": 0.0564,
+      "num_tokens": 16386481.0,
+      "reward": 0.75811767578125,
+      "reward_std": 0.0030726620461791754,
+      "rewards//mean": 0.75811767578125,
+      "rewards//std": 0.022523781284689903,
+      "step": 1896
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3794,
+      "grad_norm": 3.0030221939086914,
+      "kl": 2.376695152372122,
+      "learning_rate": 6.943427114231063e-07,
+      "loss": 0.0951,
+      "num_tokens": 16395105.0,
+      "reward": 0.78314208984375,
+      "reward_std": 0.006373615004122257,
+      "rewards//mean": 0.78314208984375,
+      "rewards//std": 0.014023732393980026,
+      "step": 1897
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3796,
+      "grad_norm": 3.121123790740967,
+      "kl": 2.3021695595234632,
+      "learning_rate": 6.94050291406296e-07,
+      "loss": 0.0921,
+      "num_tokens": 16403753.0,
+      "reward": 0.76837158203125,
+      "reward_std": 0.003544052829965949,
+      "rewards//mean": 0.76837158203125,
+      "rewards//std": 0.018528051674365997,
+      "step": 1898
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3798,
+      "grad_norm": 2.6215803623199463,
+      "kl": 1.5299625340849161,
+      "learning_rate": 6.937577932260514e-07,
+      "loss": 0.0612,
+      "num_tokens": 16412481.0,
+      "reward": 0.77008056640625,
+      "reward_std": 0.006633509416133165,
+      "rewards//mean": 0.77008056640625,
+      "rewards//std": 0.024188801646232605,
+      "step": 1899
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.38,
+      "grad_norm": 5.254921913146973,
+      "kl": 5.1301261857151985,
+      "learning_rate": 6.93465217000191e-07,
+      "loss": 0.2052,
+      "num_tokens": 16421089.0,
+      "reward": 0.73919677734375,
+      "reward_std": 0.011049970984458923,
+      "rewards//mean": 0.73919677734375,
+      "rewards//std": 0.03640343248844147,
+      "step": 1900
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3802,
+      "grad_norm": 5.4445037841796875,
+      "kl": 1.0033046416938305,
+      "learning_rate": 6.931725628465642e-07,
+      "loss": 0.0401,
+      "num_tokens": 16429721.0,
+      "reward": 0.76068115234375,
+      "reward_std": 0.0011668069055303931,
+      "rewards//mean": 0.76068115234375,
+      "rewards//std": 0.019908828660845757,
+      "step": 1901
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3804,
+      "grad_norm": 1.969037652015686,
+      "kl": 1.7568380050361156,
+      "learning_rate": 6.928798308830523e-07,
+      "loss": 0.0703,
+      "num_tokens": 16438369.0,
+      "reward": 0.75250244140625,
+      "reward_std": 0.006278123240917921,
+      "rewards//mean": 0.75250244140625,
+      "rewards//std": 0.02212301827967167,
+      "step": 1902
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3806,
+      "grad_norm": 6.894189357757568,
+      "kl": 1.9299946874380112,
+      "learning_rate": 6.925870212275676e-07,
+      "loss": 0.0772,
+      "num_tokens": 16446977.0,
+      "reward": 0.73553466796875,
+      "reward_std": 0.002975771902129054,
+      "rewards//mean": 0.73553466796875,
+      "rewards//std": 0.03099067509174347,
+      "step": 1903
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3808,
+      "grad_norm": 0.13057000935077667,
+      "kl": 0.5076147466897964,
+      "learning_rate": 6.922941339980537e-07,
+      "loss": 0.0203,
+      "num_tokens": 16455593.0,
+      "reward": 0.76409912109375,
+      "reward_std": 0.0001726334885461256,
+      "rewards//mean": 0.76409912109375,
+      "rewards//std": 0.024701541289687157,
+      "step": 1904
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.381,
+      "grad_norm": 5.440893173217773,
+      "kl": 1.5726684052497149,
+      "learning_rate": 6.920011693124856e-07,
+      "loss": 0.0629,
+      "num_tokens": 16464273.0,
+      "reward": 0.8004150390625,
+      "reward_std": 0.004623282700777054,
+      "rewards//mean": 0.8004150390625,
+      "rewards//std": 0.023966003209352493,
+      "step": 1905
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3812,
+      "grad_norm": 1.5737837553024292,
+      "kl": 1.1342873200774193,
+      "learning_rate": 6.917081272888696e-07,
+      "loss": 0.0454,
+      "num_tokens": 16472913.0,
+      "reward": 0.7923583984375,
+      "reward_std": 0.00479449238628149,
+      "rewards//mean": 0.7923583984375,
+      "rewards//std": 0.01912674866616726,
+      "step": 1906
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3814,
+      "grad_norm": 0.18740078806877136,
+      "kl": 0.5126616656780243,
+      "learning_rate": 6.914150080452428e-07,
+      "loss": 0.0205,
+      "num_tokens": 16481553.0,
+      "reward": 0.75439453125,
+      "reward_std": 0.0003452669770922512,
+      "rewards//mean": 0.75439453125,
+      "rewards//std": 0.022994112223386765,
+      "step": 1907
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3816,
+      "grad_norm": 1.2281596660614014,
+      "kl": 1.7362540028989315,
+      "learning_rate": 6.911218116996736e-07,
+      "loss": 0.0695,
+      "num_tokens": 16490177.0,
+      "reward": 0.77850341796875,
+      "reward_std": 0.003961163107305765,
+      "rewards//mean": 0.77850341796875,
+      "rewards//std": 0.017353542149066925,
+      "step": 1908
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3818,
+      "grad_norm": 4.46489143371582,
+      "kl": 4.049469344317913,
+      "learning_rate": 6.908285383702616e-07,
+      "loss": 0.162,
+      "num_tokens": 16498809.0,
+      "reward": 0.770751953125,
+      "reward_std": 0.016258426010608673,
+      "rewards//mean": 0.770751953125,
+      "rewards//std": 0.028687022626399994,
+      "step": 1909
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.382,
+      "grad_norm": 0.3052246868610382,
+      "kl": 0.5113873705267906,
+      "learning_rate": 6.905351881751371e-07,
+      "loss": 0.0205,
+      "num_tokens": 16507385.0,
+      "reward": 0.76611328125,
+      "reward_std": 0.0003452669770922512,
+      "rewards//mean": 0.76611328125,
+      "rewards//std": 0.02711247280240059,
+      "step": 1910
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3822,
+      "grad_norm": 10.212292671203613,
+      "kl": 1.0953901186585426,
+      "learning_rate": 6.902417612324615e-07,
+      "loss": 0.0438,
+      "num_tokens": 16516009.0,
+      "reward": 0.72564697265625,
+      "reward_std": 0.0009432435035705566,
+      "rewards//mean": 0.72564697265625,
+      "rewards//std": 0.021471600979566574,
+      "step": 1911
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3824,
+      "grad_norm": 0.18088702857494354,
+      "kl": 0.5196038149297237,
+      "learning_rate": 6.899482576604274e-07,
+      "loss": 0.0208,
+      "num_tokens": 16524705.0,
+      "reward": 0.72503662109375,
+      "reward_std": 0.0001726334885461256,
+      "rewards//mean": 0.72503662109375,
+      "rewards//std": 0.026808079332113266,
+      "step": 1912
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3826,
+      "grad_norm": 3.121293544769287,
+      "kl": 2.167650617659092,
+      "learning_rate": 6.896546775772576e-07,
+      "loss": 0.0867,
+      "num_tokens": 16533297.0,
+      "reward": 0.7655029296875,
+      "reward_std": 0.0039255376905202866,
+      "rewards//mean": 0.7655029296875,
+      "rewards//std": 0.024744173511862755,
+      "step": 1913
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3828,
+      "grad_norm": 3.308300733566284,
+      "kl": 3.256075270473957,
+      "learning_rate": 6.893610211012066e-07,
+      "loss": 0.1302,
+      "num_tokens": 16541953.0,
+      "reward": 0.75604248046875,
+      "reward_std": 0.007841933518648148,
+      "rewards//mean": 0.75604248046875,
+      "rewards//std": 0.023744778707623482,
+      "step": 1914
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.383,
+      "grad_norm": 2.5183308124542236,
+      "kl": 3.267112335190177,
+      "learning_rate": 6.890672883505588e-07,
+      "loss": 0.1307,
+      "num_tokens": 16550641.0,
+      "reward": 0.8157958984375,
+      "reward_std": 0.010520893149077892,
+      "rewards//mean": 0.8157958984375,
+      "rewards//std": 0.024912448599934578,
+      "step": 1915
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3832,
+      "grad_norm": 5.343947887420654,
+      "kl": 2.1912842132151127,
+      "learning_rate": 6.887734794436299e-07,
+      "loss": 0.0877,
+      "num_tokens": 16559265.0,
+      "reward": 0.80401611328125,
+      "reward_std": 0.007197076454758644,
+      "rewards//mean": 0.80401611328125,
+      "rewards//std": 0.02031005173921585,
+      "step": 1916
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 126.578125,
+      "epoch": 0.3834,
+      "grad_norm": 2.964585781097412,
+      "kl": 3.19575122743845,
+      "learning_rate": 6.884795944987661e-07,
+      "loss": 0.0998,
+      "num_tokens": 16567846.0,
+      "reward": 0.741943359375,
+      "reward_std": 0.008112197741866112,
+      "rewards//mean": 0.741943359375,
+      "rewards//std": 0.02922237664461136,
+      "step": 1917
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3836,
+      "grad_norm": 0.877312958240509,
+      "kl": 0.5779435522854328,
+      "learning_rate": 6.881856336343441e-07,
+      "loss": 0.0231,
+      "num_tokens": 16576414.0,
+      "reward": 0.77777099609375,
+      "reward_std": 0.0009345901780761778,
+      "rewards//mean": 0.77777099609375,
+      "rewards//std": 0.016054930165410042,
+      "step": 1918
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3838,
+      "grad_norm": 2.432293176651001,
+      "kl": 3.6531177069991827,
+      "learning_rate": 6.878915969687714e-07,
+      "loss": 0.1461,
+      "num_tokens": 16585126.0,
+      "reward": 0.7664794921875,
+      "reward_std": 0.009412640705704689,
+      "rewards//mean": 0.7664794921875,
+      "rewards//std": 0.02713005430996418,
+      "step": 1919
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.384,
+      "grad_norm": 4.7014665603637695,
+      "kl": 3.147923842072487,
+      "learning_rate": 6.875974846204858e-07,
+      "loss": 0.1259,
+      "num_tokens": 16593886.0,
+      "reward": 0.7637939453125,
+      "reward_std": 0.0037979367189109325,
+      "rewards//mean": 0.7637939453125,
+      "rewards//std": 0.018623093143105507,
+      "step": 1920
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3842,
+      "grad_norm": 1.4284223318099976,
+      "kl": 1.4464968033134937,
+      "learning_rate": 6.87303296707956e-07,
+      "loss": 0.0579,
+      "num_tokens": 16602454.0,
+      "reward": 0.782470703125,
+      "reward_std": 0.006412533111870289,
+      "rewards//mean": 0.782470703125,
+      "rewards//std": 0.02188645675778389,
+      "step": 1921
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3844,
+      "grad_norm": 2.7460761070251465,
+      "kl": 0.8830199912190437,
+      "learning_rate": 6.870090333496806e-07,
+      "loss": 0.0353,
+      "num_tokens": 16611174.0,
+      "reward": 0.758544921875,
+      "reward_std": 0.0026580248959362507,
+      "rewards//mean": 0.758544921875,
+      "rewards//std": 0.01050638034939766,
+      "step": 1922
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3846,
+      "grad_norm": 3.0144944190979004,
+      "kl": 0.7796740140765905,
+      "learning_rate": 6.867146946641891e-07,
+      "loss": 0.0312,
+      "num_tokens": 16619886.0,
+      "reward": 0.763427734375,
+      "reward_std": 0.0036183581687510014,
+      "rewards//mean": 0.763427734375,
+      "rewards//std": 0.01431234460324049,
+      "step": 1923
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3848,
+      "grad_norm": 1.8614685535430908,
+      "kl": 2.226287489756942,
+      "learning_rate": 6.864202807700407e-07,
+      "loss": 0.0891,
+      "num_tokens": 16628510.0,
+      "reward": 0.78741455078125,
+      "reward_std": 0.006015172693878412,
+      "rewards//mean": 0.78741455078125,
+      "rewards//std": 0.020133385434746742,
+      "step": 1924
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.385,
+      "grad_norm": 4.981801509857178,
+      "kl": 1.6397059187293053,
+      "learning_rate": 6.861257917858257e-07,
+      "loss": 0.0656,
+      "num_tokens": 16637070.0,
+      "reward": 0.76141357421875,
+      "reward_std": 0.006148449145257473,
+      "rewards//mean": 0.76141357421875,
+      "rewards//std": 0.02031899429857731,
+      "step": 1925
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3852,
+      "grad_norm": 0.3080269694328308,
+      "kl": 0.5317921955138445,
+      "learning_rate": 6.858312278301637e-07,
+      "loss": 0.0213,
+      "num_tokens": 16645686.0,
+      "reward": 0.76513671875,
+      "reward_std": 0.0003452669770922512,
+      "rewards//mean": 0.76513671875,
+      "rewards//std": 0.021693328395485878,
+      "step": 1926
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3854,
+      "grad_norm": 1.264594554901123,
+      "kl": 0.5243890956044197,
+      "learning_rate": 6.855365890217056e-07,
+      "loss": 0.021,
+      "num_tokens": 16654302.0,
+      "reward": 0.7730712890625,
+      "reward_std": 0.0006062642787583172,
+      "rewards//mean": 0.7730712890625,
+      "rewards//std": 0.020643183961510658,
+      "step": 1927
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3856,
+      "grad_norm": 1.9129774570465088,
+      "kl": 2.7641488015651703,
+      "learning_rate": 6.852418754791316e-07,
+      "loss": 0.1106,
+      "num_tokens": 16663014.0,
+      "reward": 0.83514404296875,
+      "reward_std": 0.008596815168857574,
+      "rewards//mean": 0.83514404296875,
+      "rewards//std": 0.01876833103597164,
+      "step": 1928
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3858,
+      "grad_norm": 2.6161179542541504,
+      "kl": 3.4495524652302265,
+      "learning_rate": 6.849470873211522e-07,
+      "loss": 0.138,
+      "num_tokens": 16671678.0,
+      "reward": 0.72967529296875,
+      "reward_std": 0.0054890476167202,
+      "rewards//mean": 0.72967529296875,
+      "rewards//std": 0.030932005494832993,
+      "step": 1929
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.386,
+      "grad_norm": 1.5045418739318848,
+      "kl": 1.0208391211926937,
+      "learning_rate": 6.846522246665083e-07,
+      "loss": 0.0408,
+      "num_tokens": 16680270.0,
+      "reward": 0.74993896484375,
+      "reward_std": 0.0017160075949504972,
+      "rewards//mean": 0.74993896484375,
+      "rewards//std": 0.017364878207445145,
+      "step": 1930
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3862,
+      "grad_norm": 3.578099489212036,
+      "kl": 2.0742180831730366,
+      "learning_rate": 6.843572876339704e-07,
+      "loss": 0.083,
+      "num_tokens": 16688878.0,
+      "reward": 0.74609375,
+      "reward_std": 0.007274179719388485,
+      "rewards//mean": 0.74609375,
+      "rewards//std": 0.02097237855195999,
+      "step": 1931
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3864,
+      "grad_norm": 4.720656394958496,
+      "kl": 1.0430541932582855,
+      "learning_rate": 6.840622763423391e-07,
+      "loss": 0.0417,
+      "num_tokens": 16697486.0,
+      "reward": 0.7464599609375,
+      "reward_std": 0.002421728568151593,
+      "rewards//mean": 0.7464599609375,
+      "rewards//std": 0.028507843613624573,
+      "step": 1932
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3866,
+      "grad_norm": 0.20217135548591614,
+      "kl": 0.5369813703000546,
+      "learning_rate": 6.837671909104447e-07,
+      "loss": 0.0215,
+      "num_tokens": 16706150.0,
+      "reward": 0.75970458984375,
+      "reward_std": 0.0001726334885461256,
+      "rewards//mean": 0.75970458984375,
+      "rewards//std": 0.02921505831182003,
+      "step": 1933
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3868,
+      "grad_norm": 11.511653900146484,
+      "kl": 3.0753328688442707,
+      "learning_rate": 6.834720314571479e-07,
+      "loss": 0.123,
+      "num_tokens": 16714870.0,
+      "reward": 0.7637939453125,
+      "reward_std": 0.006273890379816294,
+      "rewards//mean": 0.7637939453125,
+      "rewards//std": 0.027673648670315742,
+      "step": 1934
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.387,
+      "grad_norm": 10.808451652526855,
+      "kl": 3.7302815541625023,
+      "learning_rate": 6.831767981013388e-07,
+      "loss": 0.1492,
+      "num_tokens": 16723566.0,
+      "reward": 0.7432861328125,
+      "reward_std": 0.005238348618149757,
+      "rewards//mean": 0.7432861328125,
+      "rewards//std": 0.02662542089819908,
+      "step": 1935
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3872,
+      "grad_norm": 1.1473380327224731,
+      "kl": 1.6026993170380592,
+      "learning_rate": 6.828814909619372e-07,
+      "loss": 0.0641,
+      "num_tokens": 16732230.0,
+      "reward": 0.78302001953125,
+      "reward_std": 0.0037447321228682995,
+      "rewards//mean": 0.78302001953125,
+      "rewards//std": 0.020266776904463768,
+      "step": 1936
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3874,
+      "grad_norm": 3.704252004623413,
+      "kl": 1.4108627662062645,
+      "learning_rate": 6.82586110157893e-07,
+      "loss": 0.0564,
+      "num_tokens": 16740894.0,
+      "reward": 0.74346923828125,
+      "reward_std": 0.004684296436607838,
+      "rewards//mean": 0.74346923828125,
+      "rewards//std": 0.019930865615606308,
+      "step": 1937
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3876,
+      "grad_norm": 2.702566623687744,
+      "kl": 2.1260617580264807,
+      "learning_rate": 6.822906558081856e-07,
+      "loss": 0.085,
+      "num_tokens": 16749590.0,
+      "reward": 0.7769775390625,
+      "reward_std": 0.004325139801949263,
+      "rewards//mean": 0.7769775390625,
+      "rewards//std": 0.022463634610176086,
+      "step": 1938
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3878,
+      "grad_norm": 3.545187473297119,
+      "kl": 1.5988430865108967,
+      "learning_rate": 6.819951280318236e-07,
+      "loss": 0.064,
+      "num_tokens": 16758214.0,
+      "reward": 0.75970458984375,
+      "reward_std": 0.002550494857132435,
+      "rewards//mean": 0.75970458984375,
+      "rewards//std": 0.0240293201059103,
+      "step": 1939
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.388,
+      "grad_norm": 0.3236708343029022,
+      "kl": 0.5156267713755369,
+      "learning_rate": 6.816995269478459e-07,
+      "loss": 0.0206,
+      "num_tokens": 16766902.0,
+      "reward": 0.7744140625,
+      "reward_std": 0.0009165642550215125,
+      "rewards//mean": 0.7744140625,
+      "rewards//std": 0.021497054025530815,
+      "step": 1940
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3882,
+      "grad_norm": 8.211658477783203,
+      "kl": 3.5581136774271727,
+      "learning_rate": 6.814038526753204e-07,
+      "loss": 0.1423,
+      "num_tokens": 16775582.0,
+      "reward": 0.7431640625,
+      "reward_std": 0.005529571324586868,
+      "rewards//mean": 0.7431640625,
+      "rewards//std": 0.02930617518723011,
+      "step": 1941
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3884,
+      "grad_norm": 6.93541145324707,
+      "kl": 3.7149961553514004,
+      "learning_rate": 6.811081053333449e-07,
+      "loss": 0.1486,
+      "num_tokens": 16784198.0,
+      "reward": 0.72314453125,
+      "reward_std": 0.004664306528866291,
+      "rewards//mean": 0.72314453125,
+      "rewards//std": 0.035662416368722916,
+      "step": 1942
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3886,
+      "grad_norm": 4.992636203765869,
+      "kl": 4.106484729796648,
+      "learning_rate": 6.80812285041046e-07,
+      "loss": 0.1643,
+      "num_tokens": 16792822.0,
+      "reward": 0.78521728515625,
+      "reward_std": 0.007567324209958315,
+      "rewards//mean": 0.78521728515625,
+      "rewards//std": 0.02045191451907158,
+      "step": 1943
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3888,
+      "grad_norm": 1.497548222541809,
+      "kl": 1.7024446092545986,
+      "learning_rate": 6.805163919175806e-07,
+      "loss": 0.0681,
+      "num_tokens": 16801446.0,
+      "reward": 0.7347412109375,
+      "reward_std": 0.003278088755905628,
+      "rewards//mean": 0.7347412109375,
+      "rewards//std": 0.03186187148094177,
+      "step": 1944
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.328125,
+      "epoch": 0.389,
+      "grad_norm": 6.377845764160156,
+      "kl": 2.6406592093408108,
+      "learning_rate": 6.80220426082134e-07,
+      "loss": 0.1172,
+      "num_tokens": 16810115.0,
+      "reward": 0.770263671875,
+      "reward_std": 0.0065625980496406555,
+      "rewards//mean": 0.770263671875,
+      "rewards//std": 0.024977076798677444,
+      "step": 1945
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3892,
+      "grad_norm": 1.7635467052459717,
+      "kl": 1.741395279765129,
+      "learning_rate": 6.799243876539213e-07,
+      "loss": 0.0697,
+      "num_tokens": 16818747.0,
+      "reward": 0.7537841796875,
+      "reward_std": 0.006043876521289349,
+      "rewards//mean": 0.7537841796875,
+      "rewards//std": 0.02696438878774643,
+      "step": 1946
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 127.84375,
+      "epoch": 0.3894,
+      "grad_norm": 13.894864082336426,
+      "kl": 2.9758454263210297,
+      "learning_rate": 6.796282767521869e-07,
+      "loss": 0.1216,
+      "num_tokens": 16827361.0,
+      "reward": 0.73358154296875,
+      "reward_std": 0.0029574199579656124,
+      "rewards//mean": 0.73358154296875,
+      "rewards//std": 0.029241472482681274,
+      "step": 1947
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3896,
+      "grad_norm": 10.494483947753906,
+      "kl": 1.0923346541821957,
+      "learning_rate": 6.793320934962038e-07,
+      "loss": 0.0437,
+      "num_tokens": 16835993.0,
+      "reward": 0.77471923828125,
+      "reward_std": 0.003066399833187461,
+      "rewards//mean": 0.77471923828125,
+      "rewards//std": 0.020076163113117218,
+      "step": 1948
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3898,
+      "grad_norm": 3.937959909439087,
+      "kl": 2.0723252799361944,
+      "learning_rate": 6.790358380052751e-07,
+      "loss": 0.0829,
+      "num_tokens": 16844625.0,
+      "reward": 0.73541259765625,
+      "reward_std": 0.0021803861018270254,
+      "rewards//mean": 0.73541259765625,
+      "rewards//std": 0.02030632458627224,
+      "step": 1949
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.39,
+      "grad_norm": 5.640705585479736,
+      "kl": 3.2054260168224573,
+      "learning_rate": 6.787395103987322e-07,
+      "loss": 0.1282,
+      "num_tokens": 16853249.0,
+      "reward": 0.73291015625,
+      "reward_std": 0.006277929991483688,
+      "rewards//mean": 0.73291015625,
+      "rewards//std": 0.021760214120149612,
+      "step": 1950
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3902,
+      "grad_norm": 9.933618545532227,
+      "kl": 5.4257692992687225,
+      "learning_rate": 6.784431107959358e-07,
+      "loss": 0.217,
+      "num_tokens": 16861929.0,
+      "reward": 0.7498779296875,
+      "reward_std": 0.0079146483913064,
+      "rewards//mean": 0.7498779296875,
+      "rewards//std": 0.04172457382082939,
+      "step": 1951
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 126.953125,
+      "epoch": 0.3904,
+      "grad_norm": 4.315380096435547,
+      "kl": 2.1314827129244804,
+      "learning_rate": 6.781466393162761e-07,
+      "loss": 0.0654,
+      "num_tokens": 16870414.0,
+      "reward": 0.7310791015625,
+      "reward_std": 0.007672573905438185,
+      "rewards//mean": 0.7310791015625,
+      "rewards//std": 0.04683477804064751,
+      "step": 1952
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3906,
+      "grad_norm": 6.360894680023193,
+      "kl": 2.370349396020174,
+      "learning_rate": 6.778500960791708e-07,
+      "loss": 0.0948,
+      "num_tokens": 16879062.0,
+      "reward": 0.76434326171875,
+      "reward_std": 0.007206708192825317,
+      "rewards//mean": 0.76434326171875,
+      "rewards//std": 0.023272138088941574,
+      "step": 1953
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3908,
+      "grad_norm": 5.728469371795654,
+      "kl": 1.5349312275648117,
+      "learning_rate": 6.775534812040686e-07,
+      "loss": 0.0614,
+      "num_tokens": 16887750.0,
+      "reward": 0.76910400390625,
+      "reward_std": 0.006011077668517828,
+      "rewards//mean": 0.76910400390625,
+      "rewards//std": 0.01769132725894451,
+      "step": 1954
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.391,
+      "grad_norm": 9.019811630249023,
+      "kl": 4.6899121925234795,
+      "learning_rate": 6.772567948104452e-07,
+      "loss": 0.1876,
+      "num_tokens": 16896334.0,
+      "reward": 0.771728515625,
+      "reward_std": 0.023606345057487488,
+      "rewards//mean": 0.771728515625,
+      "rewards//std": 0.033049583435058594,
+      "step": 1955
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3912,
+      "grad_norm": 4.124979019165039,
+      "kl": 1.4003071710467339,
+      "learning_rate": 6.769600370178059e-07,
+      "loss": 0.056,
+      "num_tokens": 16905014.0,
+      "reward": 0.7510986328125,
+      "reward_std": 0.005611318163573742,
+      "rewards//mean": 0.7510986328125,
+      "rewards//std": 0.03201921656727791,
+      "step": 1956
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3914,
+      "grad_norm": 6.446751594543457,
+      "kl": 3.793982319533825,
+      "learning_rate": 6.766632079456851e-07,
+      "loss": 0.1518,
+      "num_tokens": 16913686.0,
+      "reward": 0.75048828125,
+      "reward_std": 0.009841522201895714,
+      "rewards//mean": 0.75048828125,
+      "rewards//std": 0.026259321719408035,
+      "step": 1957
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3916,
+      "grad_norm": 4.390337944030762,
+      "kl": 2.421869423240423,
+      "learning_rate": 6.76366307713645e-07,
+      "loss": 0.0969,
+      "num_tokens": 16922270.0,
+      "reward": 0.71356201171875,
+      "reward_std": 0.0046535031870007515,
+      "rewards//mean": 0.71356201171875,
+      "rewards//std": 0.03769335895776749,
+      "step": 1958
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3918,
+      "grad_norm": 7.774868965148926,
+      "kl": 2.7957709059119225,
+      "learning_rate": 6.760693364412775e-07,
+      "loss": 0.1118,
+      "num_tokens": 16930942.0,
+      "reward": 0.77545166015625,
+      "reward_std": 0.008291560225188732,
+      "rewards//mean": 0.77545166015625,
+      "rewards//std": 0.028433604165911674,
+      "step": 1959
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.392,
+      "grad_norm": 1.5700663328170776,
+      "kl": 1.0542664527893066,
+      "learning_rate": 6.757722942482022e-07,
+      "loss": 0.0422,
+      "num_tokens": 16939646.0,
+      "reward": 0.762451171875,
+      "reward_std": 0.0034526698291301727,
+      "rewards//mean": 0.762451171875,
+      "rewards//std": 0.02265854924917221,
+      "step": 1960
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3922,
+      "grad_norm": 4.8175458908081055,
+      "kl": 3.195584464818239,
+      "learning_rate": 6.754751812540679e-07,
+      "loss": 0.1278,
+      "num_tokens": 16948310.0,
+      "reward": 0.77020263671875,
+      "reward_std": 0.009702223353087902,
+      "rewards//mean": 0.77020263671875,
+      "rewards//std": 0.027835838496685028,
+      "step": 1961
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3924,
+      "grad_norm": 7.265767574310303,
+      "kl": 3.2545111440122128,
+      "learning_rate": 6.751779975785514e-07,
+      "loss": 0.1302,
+      "num_tokens": 16956958.0,
+      "reward": 0.74847412109375,
+      "reward_std": 0.00839682575315237,
+      "rewards//mean": 0.74847412109375,
+      "rewards//std": 0.03243369236588478,
+      "step": 1962
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3926,
+      "grad_norm": 6.825344562530518,
+      "kl": 2.0388757158070803,
+      "learning_rate": 6.748807433413586e-07,
+      "loss": 0.0816,
+      "num_tokens": 16965574.0,
+      "reward": 0.7994384765625,
+      "reward_std": 0.008380787447094917,
+      "rewards//mean": 0.7994384765625,
+      "rewards//std": 0.022832583636045456,
+      "step": 1963
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3928,
+      "grad_norm": 5.2122721672058105,
+      "kl": 3.83143163472414,
+      "learning_rate": 6.745834186622231e-07,
+      "loss": 0.1533,
+      "num_tokens": 16974414.0,
+      "reward": 0.7384033203125,
+      "reward_std": 0.00824748631566763,
+      "rewards//mean": 0.7384033203125,
+      "rewards//std": 0.026874415576457977,
+      "step": 1964
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.393,
+      "grad_norm": 6.726317405700684,
+      "kl": 1.003250416368246,
+      "learning_rate": 6.742860236609076e-07,
+      "loss": 0.0401,
+      "num_tokens": 16983070.0,
+      "reward": 0.75146484375,
+      "reward_std": 0.0035906690172851086,
+      "rewards//mean": 0.75146484375,
+      "rewards//std": 0.01692708395421505,
+      "step": 1965
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3932,
+      "grad_norm": 4.655847072601318,
+      "kl": 1.132183164358139,
+      "learning_rate": 6.739885584572025e-07,
+      "loss": 0.0453,
+      "num_tokens": 16991726.0,
+      "reward": 0.77154541015625,
+      "reward_std": 0.003970570396631956,
+      "rewards//mean": 0.77154541015625,
+      "rewards//std": 0.01645536907017231,
+      "step": 1966
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3934,
+      "grad_norm": 8.565656661987305,
+      "kl": 2.140916470438242,
+      "learning_rate": 6.73691023170927e-07,
+      "loss": 0.0856,
+      "num_tokens": 17000334.0,
+      "reward": 0.741943359375,
+      "reward_std": 0.003573625348508358,
+      "rewards//mean": 0.741943359375,
+      "rewards//std": 0.03848705068230629,
+      "step": 1967
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3936,
+      "grad_norm": 4.9662275314331055,
+      "kl": 0.9332614950835705,
+      "learning_rate": 6.733934179219281e-07,
+      "loss": 0.0373,
+      "num_tokens": 17008974.0,
+      "reward": 0.77008056640625,
+      "reward_std": 0.0043439012952148914,
+      "rewards//mean": 0.77008056640625,
+      "rewards//std": 0.018790094181895256,
+      "step": 1968
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3938,
+      "grad_norm": 3.461106777191162,
+      "kl": 0.5400516279041767,
+      "learning_rate": 6.730957428300811e-07,
+      "loss": 0.0216,
+      "num_tokens": 17017486.0,
+      "reward": 0.76617431640625,
+      "reward_std": 0.0008589738281443715,
+      "rewards//mean": 0.76617431640625,
+      "rewards//std": 0.02537146583199501,
+      "step": 1969
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.394,
+      "grad_norm": 5.700676918029785,
+      "kl": 1.6092449203133583,
+      "learning_rate": 6.727979980152898e-07,
+      "loss": 0.0644,
+      "num_tokens": 17026086.0,
+      "reward": 0.7872314453125,
+      "reward_std": 0.006560072768479586,
+      "rewards//mean": 0.7872314453125,
+      "rewards//std": 0.023114599287509918,
+      "step": 1970
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3942,
+      "grad_norm": 4.2400312423706055,
+      "kl": 1.0226588770747185,
+      "learning_rate": 6.725001835974852e-07,
+      "loss": 0.0409,
+      "num_tokens": 17034678.0,
+      "reward": 0.7716064453125,
+      "reward_std": 0.001833128510043025,
+      "rewards//mean": 0.7716064453125,
+      "rewards//std": 0.020774759352207184,
+      "step": 1971
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3944,
+      "grad_norm": 2.9719595909118652,
+      "kl": 1.8176025450229645,
+      "learning_rate": 6.722022996966277e-07,
+      "loss": 0.0727,
+      "num_tokens": 17043286.0,
+      "reward": 0.73687744140625,
+      "reward_std": 0.00567342946305871,
+      "rewards//mean": 0.73687744140625,
+      "rewards//std": 0.02576459012925625,
+      "step": 1972
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3946,
+      "grad_norm": 2.695848226547241,
+      "kl": 2.118714366108179,
+      "learning_rate": 6.719043464327042e-07,
+      "loss": 0.0847,
+      "num_tokens": 17051862.0,
+      "reward": 0.76287841796875,
+      "reward_std": 0.0035351249389350414,
+      "rewards//mean": 0.76287841796875,
+      "rewards//std": 0.021459612995386124,
+      "step": 1973
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3948,
+      "grad_norm": 2.256169319152832,
+      "kl": 2.1824787370860577,
+      "learning_rate": 6.716063239257306e-07,
+      "loss": 0.0873,
+      "num_tokens": 17060526.0,
+      "reward": 0.755859375,
+      "reward_std": 0.001942233182489872,
+      "rewards//mean": 0.755859375,
+      "rewards//std": 0.035297200083732605,
+      "step": 1974
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.395,
+      "grad_norm": 2.6194136142730713,
+      "kl": 0.9225284308195114,
+      "learning_rate": 6.713082322957502e-07,
+      "loss": 0.0369,
+      "num_tokens": 17069118.0,
+      "reward": 0.77252197265625,
+      "reward_std": 0.0016100658103823662,
+      "rewards//mean": 0.77252197265625,
+      "rewards//std": 0.020419321954250336,
+      "step": 1975
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3952,
+      "grad_norm": 5.163909435272217,
+      "kl": 3.525698345154524,
+      "learning_rate": 6.710100716628344e-07,
+      "loss": 0.141,
+      "num_tokens": 17077790.0,
+      "reward": 0.76458740234375,
+      "reward_std": 0.006846260279417038,
+      "rewards//mean": 0.76458740234375,
+      "rewards//std": 0.03611956164240837,
+      "step": 1976
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3954,
+      "grad_norm": 1.7235068082809448,
+      "kl": 0.5247271433472633,
+      "learning_rate": 6.70711842147082e-07,
+      "loss": 0.021,
+      "num_tokens": 17086342.0,
+      "reward": 0.7650146484375,
+      "reward_std": 0.0003452669770922512,
+      "rewards//mean": 0.7650146484375,
+      "rewards//std": 0.019139407202601433,
+      "step": 1977
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3956,
+      "grad_norm": 4.16033411026001,
+      "kl": 0.6964804530143738,
+      "learning_rate": 6.704135438686203e-07,
+      "loss": 0.0279,
+      "num_tokens": 17094966.0,
+      "reward": 0.77239990234375,
+      "reward_std": 0.0036463579162955284,
+      "rewards//mean": 0.77239990234375,
+      "rewards//std": 0.02306501939892769,
+      "step": 1978
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3958,
+      "grad_norm": 2.280411720275879,
+      "kl": 1.7261448204517365,
+      "learning_rate": 6.701151769476032e-07,
+      "loss": 0.069,
+      "num_tokens": 17103566.0,
+      "reward": 0.77471923828125,
+      "reward_std": 0.004031552001833916,
+      "rewards//mean": 0.77471923828125,
+      "rewards//std": 0.02696516178548336,
+      "step": 1979
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.396,
+      "grad_norm": 7.489660739898682,
+      "kl": 1.3034958392381668,
+      "learning_rate": 6.698167415042134e-07,
+      "loss": 0.0521,
+      "num_tokens": 17112270.0,
+      "reward": 0.7337646484375,
+      "reward_std": 0.001598489936441183,
+      "rewards//mean": 0.7337646484375,
+      "rewards//std": 0.02923351153731346,
+      "step": 1980
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3962,
+      "grad_norm": 8.335091590881348,
+      "kl": 3.803154118359089,
+      "learning_rate": 6.695182376586602e-07,
+      "loss": 0.1521,
+      "num_tokens": 17121046.0,
+      "reward": 0.76043701171875,
+      "reward_std": 0.00808590929955244,
+      "rewards//mean": 0.76043701171875,
+      "rewards//std": 0.03781525045633316,
+      "step": 1981
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3964,
+      "grad_norm": 4.71075439453125,
+      "kl": 2.199516534805298,
+      "learning_rate": 6.692196655311814e-07,
+      "loss": 0.088,
+      "num_tokens": 17129606.0,
+      "reward": 0.7464599609375,
+      "reward_std": 0.005811984650790691,
+      "rewards//mean": 0.7464599609375,
+      "rewards//std": 0.027372246608138084,
+      "step": 1982
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3966,
+      "grad_norm": 20.35379981994629,
+      "kl": 6.530514575541019,
+      "learning_rate": 6.689210252420415e-07,
+      "loss": 0.2612,
+      "num_tokens": 17138302.0,
+      "reward": 0.73248291015625,
+      "reward_std": 0.008087247610092163,
+      "rewards//mean": 0.73248291015625,
+      "rewards//std": 0.035943109542131424,
+      "step": 1983
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3968,
+      "grad_norm": 41.484493255615234,
+      "kl": 9.794805243611336,
+      "learning_rate": 6.686223169115327e-07,
+      "loss": 0.3918,
+      "num_tokens": 17147094.0,
+      "reward": 0.73590087890625,
+      "reward_std": 0.0081851901486516,
+      "rewards//mean": 0.73590087890625,
+      "rewards//std": 0.03236641362309456,
+      "step": 1984
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.397,
+      "grad_norm": 2.531568765640259,
+      "kl": 1.0259710513055325,
+      "learning_rate": 6.683235406599749e-07,
+      "loss": 0.041,
+      "num_tokens": 17155750.0,
+      "reward": 0.74365234375,
+      "reward_std": 0.004134616814553738,
+      "rewards//mean": 0.74365234375,
+      "rewards//std": 0.02501220442354679,
+      "step": 1985
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3972,
+      "grad_norm": 21.211650848388672,
+      "kl": 4.83174666762352,
+      "learning_rate": 6.68024696607715e-07,
+      "loss": 0.1933,
+      "num_tokens": 17164486.0,
+      "reward": 0.78106689453125,
+      "reward_std": 0.00467984564602375,
+      "rewards//mean": 0.78106689453125,
+      "rewards//std": 0.022792354226112366,
+      "step": 1986
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3974,
+      "grad_norm": 0.18116259574890137,
+      "kl": 0.5025690570473671,
+      "learning_rate": 6.677257848751276e-07,
+      "loss": 0.0201,
+      "num_tokens": 17173118.0,
+      "reward": 0.80072021484375,
+      "reward_std": 0.0001726334885461256,
+      "rewards//mean": 0.80072021484375,
+      "rewards//std": 0.016840901225805283,
+      "step": 1987
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3976,
+      "grad_norm": 2.183115243911743,
+      "kl": 2.132648713886738,
+      "learning_rate": 6.674268055826138e-07,
+      "loss": 0.0853,
+      "num_tokens": 17181830.0,
+      "reward": 0.77362060546875,
+      "reward_std": 0.006493935827165842,
+      "rewards//mean": 0.77362060546875,
+      "rewards//std": 0.02627711370587349,
+      "step": 1988
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3978,
+      "grad_norm": 1.2820903062820435,
+      "kl": 1.1089227385818958,
+      "learning_rate": 6.671277588506029e-07,
+      "loss": 0.0444,
+      "num_tokens": 17190510.0,
+      "reward": 0.7415771484375,
+      "reward_std": 0.0029527952428907156,
+      "rewards//mean": 0.7415771484375,
+      "rewards//std": 0.03049067221581936,
+      "step": 1989
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.398,
+      "grad_norm": 4.301590442657471,
+      "kl": 3.203248431906104,
+      "learning_rate": 6.668286447995507e-07,
+      "loss": 0.1281,
+      "num_tokens": 17199142.0,
+      "reward": 0.7623291015625,
+      "reward_std": 0.00420347461476922,
+      "rewards//mean": 0.7623291015625,
+      "rewards//std": 0.02261943183839321,
+      "step": 1990
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3982,
+      "grad_norm": 1.2312685251235962,
+      "kl": 1.366452895104885,
+      "learning_rate": 6.665294635499403e-07,
+      "loss": 0.0547,
+      "num_tokens": 17207790.0,
+      "reward": 0.77581787109375,
+      "reward_std": 0.0036111832596361637,
+      "rewards//mean": 0.77581787109375,
+      "rewards//std": 0.018422355875372887,
+      "step": 1991
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3984,
+      "grad_norm": 2.7833945751190186,
+      "kl": 2.5449508111923933,
+      "learning_rate": 6.66230215222282e-07,
+      "loss": 0.1018,
+      "num_tokens": 17216406.0,
+      "reward": 0.77215576171875,
+      "reward_std": 0.007916904985904694,
+      "rewards//mean": 0.77215576171875,
+      "rewards//std": 0.031095029786229134,
+      "step": 1992
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3986,
+      "grad_norm": 2.739675283432007,
+      "kl": 2.807829365134239,
+      "learning_rate": 6.659308999371129e-07,
+      "loss": 0.1123,
+      "num_tokens": 17225070.0,
+      "reward": 0.7308349609375,
+      "reward_std": 0.005479984916746616,
+      "rewards//mean": 0.7308349609375,
+      "rewards//std": 0.02748703584074974,
+      "step": 1993
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3988,
+      "grad_norm": 3.9408509731292725,
+      "kl": 2.529978759586811,
+      "learning_rate": 6.65631517814997e-07,
+      "loss": 0.1012,
+      "num_tokens": 17233710.0,
+      "reward": 0.7501220703125,
+      "reward_std": 0.005697350949048996,
+      "rewards//mean": 0.7501220703125,
+      "rewards//std": 0.02913184091448784,
+      "step": 1994
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.399,
+      "grad_norm": 0.9195917844772339,
+      "kl": 1.3602860979735851,
+      "learning_rate": 6.653320689765256e-07,
+      "loss": 0.0544,
+      "num_tokens": 17242334.0,
+      "reward": 0.7642822265625,
+      "reward_std": 0.0020170093048363924,
+      "rewards//mean": 0.7642822265625,
+      "rewards//std": 0.017206918448209763,
+      "step": 1995
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3992,
+      "grad_norm": 3.3152499198913574,
+      "kl": 2.117432627826929,
+      "learning_rate": 6.650325535423166e-07,
+      "loss": 0.0847,
+      "num_tokens": 17250950.0,
+      "reward": 0.74365234375,
+      "reward_std": 0.002731853164732456,
+      "rewards//mean": 0.74365234375,
+      "rewards//std": 0.019054194912314415,
+      "step": 1996
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3994,
+      "grad_norm": 12.34266471862793,
+      "kl": 4.6731117479503155,
+      "learning_rate": 6.647329716330147e-07,
+      "loss": 0.1869,
+      "num_tokens": 17259566.0,
+      "reward": 0.7325439453125,
+      "reward_std": 0.0050992234610021114,
+      "rewards//mean": 0.7325439453125,
+      "rewards//std": 0.03254812955856323,
+      "step": 1997
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3996,
+      "grad_norm": 3.4830985069274902,
+      "kl": 3.159088408574462,
+      "learning_rate": 6.644333233692916e-07,
+      "loss": 0.1264,
+      "num_tokens": 17268366.0,
+      "reward": 0.77423095703125,
+      "reward_std": 0.009905059821903706,
+      "rewards//mean": 0.77423095703125,
+      "rewards//std": 0.02363487333059311,
+      "step": 1998
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.3998,
+      "grad_norm": 2.4505362510681152,
+      "kl": 0.5356312431395054,
+      "learning_rate": 6.641336088718456e-07,
+      "loss": 0.0214,
+      "num_tokens": 17277014.0,
+      "reward": 0.7716064453125,
+      "reward_std": 0.0003452669770922512,
+      "rewards//mean": 0.7716064453125,
+      "rewards//std": 0.02353000082075596,
+      "step": 1999
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.4,
+      "grad_norm": 2.995344638824463,
+      "kl": 1.5457897055894136,
+      "learning_rate": 6.638338282614014e-07,
+      "loss": 0.0618,
+      "num_tokens": 17285654.0,
+      "reward": 0.73614501953125,
+      "reward_std": 0.005006371531635523,
+      "rewards//mean": 0.73614501953125,
+      "rewards//std": 0.03228633850812912,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 5000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}