diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.0015,
+  "epoch": 2e-05,
   "eval_steps": 500,
-  "global_step": 75,
+  "global_step": 1,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -16,2478 +16,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 638.0,
-      "completions/max_terminated_length": 638.0,
-      "completions/mean_length": 39.28125,
-      "completions/mean_terminated_length": 39.28125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.39564167940989137,
+      "completions/max_length": 1087.0,
+      "completions/max_terminated_length": 1087.0,
+      "completions/mean_length": 981.25,
+      "completions/mean_terminated_length": 981.25,
+      "completions/min_length": 688.0,
+      "completions/min_terminated_length": 688.0,
+      "entropy": 0.16603230126202106,
       "epoch": 2e-05,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.6560407876968384,
+      "grad_norm": 2.348139762878418,
       "kl": 0.0,
       "learning_rate": 0.0,
-      "loss": -0.0348,
-      "num_tokens": 45390.0,
-      "reward": -0.9619714617729187,
-      "reward_std": 0.2875919044017792,
-      "rewards/rollout_reward_func/mean": -0.9619714617729187,
-      "rewards/rollout_reward_func/std": 0.3140026926994324,
-      "sampling/importance_sampling_ratio/max": 2.1279656887054443,
-      "sampling/importance_sampling_ratio/mean": 1.098207712173462,
-      "sampling/importance_sampling_ratio/min": 0.7664512991905212,
-      "sampling/sampling_logp_difference/max": 0.7476255893707275,
-      "sampling/sampling_logp_difference/mean": 0.041369449347257614,
+      "loss": -0.0197,
+      "num_tokens": 65998.0,
+      "reward": -0.5384306311607361,
+      "reward_std": 0.15059955418109894,
+      "rewards/rollout_reward_func/mean": -0.5384306311607361,
+      "rewards/rollout_reward_func/std": 0.16172325611114502,
+      "sampling/importance_sampling_ratio/max": 1.562467336654663,
+      "sampling/importance_sampling_ratio/mean": 1.0033960342407227,
+      "sampling/importance_sampling_ratio/min": 0.6933757066726685,
+      "sampling/sampling_logp_difference/max": 0.44626617431640625,
+      "sampling/sampling_logp_difference/mean": 0.020405184477567673,
       "step": 1,
-      "step_time": 12.809986616000174
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 638.0,
-      "completions/max_terminated_length": 638.0,
-      "completions/mean_length": 78.25,
-      "completions/mean_terminated_length": 78.25,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.4294010205194354,
-      "epoch": 4e-05,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 2.1191000938415527,
-      "kl": 0.0,
-      "learning_rate": 2.8571428571428573e-06,
-      "loss": -0.0036,
-      "num_tokens": 95751.0,
-      "reward": -1.0162220001220703,
-      "reward_std": 0.11686795204877853,
-      "rewards/rollout_reward_func/mean": -1.0162220001220703,
-      "rewards/rollout_reward_func/std": 0.18659980595111847,
-      "sampling/importance_sampling_ratio/max": 1.494768500328064,
-      "sampling/importance_sampling_ratio/mean": 0.9892289638519287,
-      "sampling/importance_sampling_ratio/min": 0.34746432304382324,
-      "sampling/sampling_logp_difference/max": 1.0969908237457275,
-      "sampling/sampling_logp_difference/mean": 0.050151824951171875,
-      "step": 2,
-      "step_time": 11.691297715000132
-    },
-    {
-      "clip_ratio/high_max": 0.0416666679084301,
-      "clip_ratio/high_mean": 0.02083333395421505,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.02083333395421505,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 459.0,
-      "completions/max_terminated_length": 459.0,
-      "completions/mean_length": 27.4375,
-      "completions/mean_terminated_length": 27.4375,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.368882168084383,
-      "epoch": 6e-05,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.375580072402954,
-      "kl": 0.004193210355879273,
-      "learning_rate": 5.7142857142857145e-06,
-      "loss": 0.0066,
-      "num_tokens": 141464.0,
-      "reward": -1.0646312236785889,
-      "reward_std": 0.03747585415840149,
-      "rewards/rollout_reward_func/mean": -1.0646312236785889,
-      "rewards/rollout_reward_func/std": 0.038347113877534866,
-      "sampling/importance_sampling_ratio/max": 1.3222557306289673,
-      "sampling/importance_sampling_ratio/mean": 0.9890090227127075,
-      "sampling/importance_sampling_ratio/min": 0.5562543869018555,
-      "sampling/sampling_logp_difference/max": 0.4829235076904297,
-      "sampling/sampling_logp_difference/mean": 0.039045777171850204,
-      "step": 3,
-      "step_time": 9.59587805300032
-    },
-    {
-      "clip_ratio/high_max": 0.07500000204890966,
-      "clip_ratio/high_mean": 0.04791666800156236,
-      "clip_ratio/low_mean": 0.031250000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.07916666939854622,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 677.0,
-      "completions/max_terminated_length": 677.0,
-      "completions/mean_length": 38.0,
-      "completions/mean_terminated_length": 38.0,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.4608576577156782,
-      "epoch": 8e-05,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.7839480638504028,
-      "kl": 0.003663932517952162,
-      "learning_rate": 8.571428571428573e-06,
-      "loss": -0.0009,
-      "num_tokens": 192265.0,
-      "reward": -1.0257494449615479,
-      "reward_std": 0.12461724132299423,
-      "rewards/rollout_reward_func/mean": -1.0257494449615479,
-      "rewards/rollout_reward_func/std": 0.2001343071460724,
-      "sampling/importance_sampling_ratio/max": 1.4480887651443481,
-      "sampling/importance_sampling_ratio/mean": 0.9914816617965698,
-      "sampling/importance_sampling_ratio/min": 0.42514756321907043,
-      "sampling/sampling_logp_difference/max": 0.6030776500701904,
-      "sampling/sampling_logp_difference/mean": 0.049290064722299576,
-      "step": 4,
-      "step_time": 11.246127179000041
-    },
-    {
-      "clip_ratio/high_max": 0.0416666679084301,
-      "clip_ratio/high_mean": 0.02083333395421505,
-      "clip_ratio/low_mean": 0.015625,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.03645833395421505,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 677.0,
-      "completions/max_terminated_length": 677.0,
-      "completions/mean_length": 24.03125,
-      "completions/mean_terminated_length": 24.03125,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.44877374544739723,
-      "epoch": 0.0001,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.4991412162780762,
-      "kl": 0.009261666031306959,
-      "learning_rate": 1.1428571428571429e-05,
-      "loss": -0.0031,
-      "num_tokens": 239102.0,
-      "reward": -1.0257375240325928,
-      "reward_std": 0.1197652742266655,
-      "rewards/rollout_reward_func/mean": -1.0257375240325928,
-      "rewards/rollout_reward_func/std": 0.19252590835094452,
-      "sampling/importance_sampling_ratio/max": 1.2955385446548462,
-      "sampling/importance_sampling_ratio/mean": 0.9851011037826538,
-      "sampling/importance_sampling_ratio/min": 0.37779325246810913,
-      "sampling/sampling_logp_difference/max": 0.965994119644165,
-      "sampling/sampling_logp_difference/mean": 0.04231194779276848,
-      "step": 5,
-      "step_time": 10.247773965999954
-    },
-    {
-      "clip_ratio/high_max": 0.02083333395421505,
-      "clip_ratio/high_mean": 0.010416666977107525,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.02083333395421505,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 518.0,
-      "completions/max_terminated_length": 518.0,
-      "completions/mean_length": 47.21875,
-      "completions/mean_terminated_length": 47.21875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.3389640529640019,
-      "epoch": 0.00012,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 5.797247886657715,
-      "kl": 0.012303851813840083,
-      "learning_rate": 1.4285714285714285e-05,
-      "loss": -0.006,
-      "num_tokens": 287416.0,
-      "reward": -0.9584284424781799,
-      "reward_std": 0.2973359227180481,
-      "rewards/rollout_reward_func/mean": -0.9584284424781799,
-      "rewards/rollout_reward_func/std": 0.4072912335395813,
-      "sampling/importance_sampling_ratio/max": 1.5715361833572388,
-      "sampling/importance_sampling_ratio/mean": 0.9542790651321411,
-      "sampling/importance_sampling_ratio/min": 0.6721829771995544,
-      "sampling/sampling_logp_difference/max": 0.45198535919189453,
-      "sampling/sampling_logp_difference/mean": 0.04381281137466431,
-      "step": 6,
-      "step_time": 10.943097969000064
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.031250000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.031250000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 704.0,
-      "completions/max_terminated_length": 704.0,
-      "completions/mean_length": 106.875,
-      "completions/mean_terminated_length": 106.875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.38349559996277094,
-      "epoch": 0.00014,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 5.084864616394043,
-      "kl": 0.06875983771169558,
-      "learning_rate": 1.7142857142857145e-05,
-      "loss": -0.1721,
-      "num_tokens": 339488.0,
-      "reward": -1.0205574035644531,
-      "reward_std": 0.12819266319274902,
-      "rewards/rollout_reward_func/mean": -1.0205574035644531,
-      "rewards/rollout_reward_func/std": 0.20544660091400146,
-      "sampling/importance_sampling_ratio/max": 2.900042772293091,
-      "sampling/importance_sampling_ratio/mean": 0.9976917505264282,
-      "sampling/importance_sampling_ratio/min": 0.3586415648460388,
-      "sampling/sampling_logp_difference/max": 0.9816085696220398,
-      "sampling/sampling_logp_difference/mean": 0.05562879890203476,
-      "step": 7,
-      "step_time": 10.513272228999881
-    },
-    {
-      "clip_ratio/high_max": 0.07500000204890966,
-      "clip_ratio/high_mean": 0.03750000102445483,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.03750000102445483,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 538.0,
-      "completions/max_terminated_length": 538.0,
-      "completions/mean_length": 46.90625,
-      "completions/mean_terminated_length": 46.90625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.43156392499804497,
-      "epoch": 0.00016,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.9030671119689941,
-      "kl": 0.05070029087073635,
-      "learning_rate": 2e-05,
-      "loss": -0.0129,
-      "num_tokens": 388162.0,
-      "reward": -0.9287885427474976,
-      "reward_std": 0.39020878076553345,
-      "rewards/rollout_reward_func/mean": -0.9287885427474976,
-      "rewards/rollout_reward_func/std": 0.5155612230300903,
-      "sampling/importance_sampling_ratio/max": 1.4499342441558838,
-      "sampling/importance_sampling_ratio/mean": 0.9176421165466309,
-      "sampling/importance_sampling_ratio/min": 0.41842636466026306,
-      "sampling/sampling_logp_difference/max": 0.7849738001823425,
-      "sampling/sampling_logp_difference/mean": 0.0675830990076065,
-      "step": 8,
-      "step_time": 10.969506959999876
-    },
-    {
-      "clip_ratio/high_max": 0.00390625,
-      "clip_ratio/high_mean": 0.001953125,
-      "clip_ratio/low_mean": 0.026041667442768812,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.027994792442768812,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 639.0,
-      "completions/max_terminated_length": 639.0,
-      "completions/mean_length": 200.5625,
-      "completions/mean_terminated_length": 200.5625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.321466947265435,
-      "epoch": 0.00018,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.0046943426132202,
-      "kl": 0.35286774128326215,
-      "learning_rate": 2.2857142857142858e-05,
-      "loss": -0.0073,
-      "num_tokens": 443637.0,
-      "reward": -0.9580092430114746,
-      "reward_std": 0.29512760043144226,
-      "rewards/rollout_reward_func/mean": -0.9580092430114746,
-      "rewards/rollout_reward_func/std": 0.3135679066181183,
-      "sampling/importance_sampling_ratio/max": 1.3844846487045288,
-      "sampling/importance_sampling_ratio/mean": 0.9317118525505066,
-      "sampling/importance_sampling_ratio/min": 0.28221532702445984,
-      "sampling/sampling_logp_difference/max": 1.2594640254974365,
-      "sampling/sampling_logp_difference/mean": 0.05695861577987671,
-      "step": 9,
-      "step_time": 10.883997247999787
-    },
-    {
-      "clip_ratio/high_max": 0.03854166716337204,
-      "clip_ratio/high_mean": 0.01927083358168602,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.029687500558793545,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 293.875,
-      "completions/mean_terminated_length": 293.875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.302602864059736,
-      "epoch": 0.0002,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.2237683534622192,
-      "kl": 0.6169689407106489,
-      "learning_rate": 2.5714285714285714e-05,
-      "loss": -0.0205,
-      "num_tokens": 502287.0,
-      "reward": -0.8612995147705078,
-      "reward_std": 0.4479275345802307,
-      "rewards/rollout_reward_func/mean": -0.8612995147705078,
-      "rewards/rollout_reward_func/std": 0.4341149628162384,
-      "sampling/importance_sampling_ratio/max": 1.8056789636611938,
-      "sampling/importance_sampling_ratio/mean": 0.9470912218093872,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 4.063522815704346,
-      "sampling/sampling_logp_difference/mean": 0.11726777255535126,
-      "step": 10,
-      "step_time": 12.200401258000284
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0052083334885537624,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0052083334885537624,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 315.28125,
-      "completions/mean_terminated_length": 315.28125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.19023864425253123,
-      "epoch": 0.00022,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.692126989364624,
-      "kl": 1.2167398069868796,
-      "learning_rate": 2.857142857142857e-05,
-      "loss": -0.0612,
-      "num_tokens": 559927.0,
-      "reward": -0.8648344278335571,
-      "reward_std": 0.4261757731437683,
-      "rewards/rollout_reward_func/mean": -0.8648344278335571,
-      "rewards/rollout_reward_func/std": 0.42738500237464905,
-      "sampling/importance_sampling_ratio/max": 2.22552227973938,
-      "sampling/importance_sampling_ratio/mean": 0.9920452833175659,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 1.688279628753662,
-      "sampling/sampling_logp_difference/mean": 0.08655641227960587,
-      "step": 11,
-      "step_time": 12.600086301999909
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0062500000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 712.0,
-      "completions/max_terminated_length": 712.0,
-      "completions/mean_length": 256.46875,
-      "completions/mean_terminated_length": 256.46875,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.12821872769563925,
-      "epoch": 0.00024,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.1939445734024048,
-      "kl": 0.6710226865034201,
-      "learning_rate": 3.142857142857143e-05,
-      "loss": -0.0738,
-      "num_tokens": 612121.0,
-      "reward": -0.8006033897399902,
-      "reward_std": 0.5579005479812622,
-      "rewards/rollout_reward_func/mean": -0.8006033897399902,
-      "rewards/rollout_reward_func/std": 0.5530418157577515,
-      "sampling/importance_sampling_ratio/max": 2.040736198425293,
-      "sampling/importance_sampling_ratio/mean": 0.9843668341636658,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 2.237351894378662,
-      "sampling/sampling_logp_difference/mean": 0.07982916384935379,
-      "step": 12,
-      "step_time": 11.606591519000062
-    },
-    {
-      "clip_ratio/high_max": 0.02083333395421505,
-      "clip_ratio/high_mean": 0.010416666977107525,
-      "clip_ratio/low_mean": 0.009722222341224551,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.020138889318332076,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 349.0625,
-      "completions/mean_terminated_length": 349.0625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.15199650880458648,
-      "epoch": 0.00026,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.810089111328125,
-      "kl": 0.6918285206593282,
-      "learning_rate": 3.428571428571429e-05,
-      "loss": -0.0022,
-      "num_tokens": 666473.0,
-      "reward": -0.5997135639190674,
-      "reward_std": 0.6681541204452515,
-      "rewards/rollout_reward_func/mean": -0.5997135639190674,
-      "rewards/rollout_reward_func/std": 0.6582958698272705,
-      "sampling/importance_sampling_ratio/max": 2.443354368209839,
-      "sampling/importance_sampling_ratio/mean": 0.996698260307312,
-      "sampling/importance_sampling_ratio/min": 0.0020917414221912622,
-      "sampling/sampling_logp_difference/max": 6.307608604431152,
-      "sampling/sampling_logp_difference/mean": 0.11165391653776169,
-      "step": 13,
-      "step_time": 11.55509544199981
-    },
-    {
-      "clip_ratio/high_max": 0.02500000037252903,
-      "clip_ratio/high_mean": 0.012500000186264515,
-      "clip_ratio/low_mean": 0.01666666753590107,
-      "clip_ratio/low_min": 0.012500000186264515,
-      "clip_ratio/region_mean": 0.029166667722165585,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 336.40625,
-      "completions/mean_terminated_length": 336.40625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.1205911443757941,
-      "epoch": 0.00028,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.44033682346344,
-      "kl": 1.3907820242457092,
-      "learning_rate": 3.7142857142857143e-05,
-      "loss": -0.0368,
-      "num_tokens": 724590.0,
-      "reward": -0.7913841009140015,
-      "reward_std": 0.4581625163555145,
-      "rewards/rollout_reward_func/mean": -0.7913841009140015,
-      "rewards/rollout_reward_func/std": 0.5302081108093262,
-      "sampling/importance_sampling_ratio/max": 1.4406403303146362,
-      "sampling/importance_sampling_ratio/mean": 0.9338162541389465,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 2.373154878616333,
-      "sampling/sampling_logp_difference/mean": 0.05714879930019379,
-      "step": 14,
-      "step_time": 11.826911633000122
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.016964286100119352,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.02321428619325161,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 694.0,
-      "completions/max_terminated_length": 694.0,
-      "completions/mean_length": 320.25,
-      "completions/mean_terminated_length": 320.25,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.11132675514090806,
-      "epoch": 0.0003,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.0571273565292358,
-      "kl": 0.6359303884673864,
-      "learning_rate": 4e-05,
-      "loss": -0.0052,
-      "num_tokens": 778055.0,
-      "reward": -0.7417740821838379,
-      "reward_std": 0.5058959722518921,
-      "rewards/rollout_reward_func/mean": -0.7417740821838379,
-      "rewards/rollout_reward_func/std": 0.621690034866333,
-      "sampling/importance_sampling_ratio/max": 2.0310840606689453,
-      "sampling/importance_sampling_ratio/mean": 0.9988756775856018,
-      "sampling/importance_sampling_ratio/min": 0.11469075083732605,
-      "sampling/sampling_logp_difference/max": 2.255527973175049,
-      "sampling/sampling_logp_difference/mean": 0.06771025061607361,
-      "step": 15,
-      "step_time": 11.486071078000236
-    },
-    {
-      "clip_ratio/high_max": 0.053125000558793545,
-      "clip_ratio/high_mean": 0.03281250083819032,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.03281250083819032,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 704.0,
-      "completions/max_terminated_length": 704.0,
-      "completions/mean_length": 274.6875,
-      "completions/mean_terminated_length": 274.6875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.06733570974029135,
-      "epoch": 0.00032,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.9541396498680115,
-      "kl": 0.7362577195162885,
-      "learning_rate": 4.2857142857142856e-05,
-      "loss": -0.0012,
-      "num_tokens": 833437.0,
-      "reward": -0.6857098340988159,
-      "reward_std": 0.7079445719718933,
-      "rewards/rollout_reward_func/mean": -0.6857098340988159,
-      "rewards/rollout_reward_func/std": 0.6899812817573547,
-      "sampling/importance_sampling_ratio/max": 1.155164122581482,
-      "sampling/importance_sampling_ratio/mean": 0.8961647748947144,
-      "sampling/importance_sampling_ratio/min": 0.013389181345701218,
-      "sampling/sampling_logp_difference/max": 2.804170608520508,
-      "sampling/sampling_logp_difference/mean": 0.06406959146261215,
-      "step": 16,
-      "step_time": 11.482106883000142
-    },
-    {
-      "clip_ratio/high_max": 0.04062500037252903,
-      "clip_ratio/high_mean": 0.020312500186264515,
-      "clip_ratio/low_mean": 0.012500000186264515,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.03281250037252903,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 382.5,
-      "completions/mean_terminated_length": 382.5,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.11045419698348269,
-      "epoch": 0.00034,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.9699212312698364,
-      "kl": 2.2062103087082505,
-      "learning_rate": 4.5714285714285716e-05,
-      "loss": -0.0046,
-      "num_tokens": 891074.0,
-      "reward": -0.616033673286438,
-      "reward_std": 0.612152099609375,
-      "rewards/rollout_reward_func/mean": -0.616033673286438,
-      "rewards/rollout_reward_func/std": 0.6434445977210999,
-      "sampling/importance_sampling_ratio/max": 1.7174687385559082,
-      "sampling/importance_sampling_ratio/mean": 0.8949162364006042,
-      "sampling/importance_sampling_ratio/min": 0.0025639955420047045,
-      "sampling/sampling_logp_difference/max": 5.869161128997803,
-      "sampling/sampling_logp_difference/mean": 0.10544054210186005,
-      "step": 17,
-      "step_time": 11.477599754000494
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.0062500000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.012500000186264515,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 722.0,
-      "completions/max_terminated_length": 722.0,
-      "completions/mean_length": 407.65625,
-      "completions/mean_terminated_length": 407.65625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.09873936581425369,
-      "epoch": 0.00036,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.0079271793365479,
-      "kl": 0.6991606876254082,
-      "learning_rate": 4.8571428571428576e-05,
-      "loss": 0.0307,
-      "num_tokens": 952477.0,
-      "reward": -0.4275363087654114,
-      "reward_std": 0.9017512798309326,
-      "rewards/rollout_reward_func/mean": -0.4275363087654114,
-      "rewards/rollout_reward_func/std": 0.8709455728530884,
-      "sampling/importance_sampling_ratio/max": 1.6146125793457031,
-      "sampling/importance_sampling_ratio/mean": 0.954972505569458,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 2.63460111618042,
-      "sampling/sampling_logp_difference/mean": 0.05541383847594261,
-      "step": 18,
-      "step_time": 11.420186637000029
-    },
-    {
-      "clip_ratio/high_max": 0.02500000037252903,
-      "clip_ratio/high_mean": 0.012500000186264515,
-      "clip_ratio/low_mean": 0.01490384666249156,
-      "clip_ratio/low_min": 0.004807692486792803,
-      "clip_ratio/region_mean": 0.027403846848756075,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 365.0625,
-      "completions/mean_terminated_length": 365.0625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.07543634981266223,
-      "epoch": 0.00038,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.45864737033843994,
-      "kl": 0.8905722848139703,
-      "learning_rate": 5.142857142857143e-05,
-      "loss": -0.0106,
-      "num_tokens": 1010821.0,
-      "reward": -0.16911554336547852,
-      "reward_std": 0.9249227046966553,
-      "rewards/rollout_reward_func/mean": -0.16911554336547852,
-      "rewards/rollout_reward_func/std": 0.9291653037071228,
-      "sampling/importance_sampling_ratio/max": 2.365478038787842,
-      "sampling/importance_sampling_ratio/mean": 1.0010688304901123,
-      "sampling/importance_sampling_ratio/min": 0.14363108575344086,
-      "sampling/sampling_logp_difference/max": 2.3417558670043945,
-      "sampling/sampling_logp_difference/mean": 0.041936248540878296,
-      "step": 19,
-      "step_time": 12.218254700000102
-    },
-    {
-      "clip_ratio/high_max": 0.04543269285932183,
-      "clip_ratio/high_mean": 0.022716346429660916,
-      "clip_ratio/low_mean": 0.0020833334419876337,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.02479967987164855,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 712.0,
-      "completions/max_terminated_length": 712.0,
-      "completions/mean_length": 317.78125,
-      "completions/mean_terminated_length": 317.78125,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.09119368118990678,
-      "epoch": 0.0004,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.48093339800834656,
-      "kl": 0.920582022437884,
-      "learning_rate": 5.428571428571428e-05,
-      "loss": -0.0405,
-      "num_tokens": 1070702.0,
-      "reward": -0.2649157643318176,
-      "reward_std": 0.7187560796737671,
-      "rewards/rollout_reward_func/mean": -0.2649157643318176,
-      "rewards/rollout_reward_func/std": 0.9145097732543945,
-      "sampling/importance_sampling_ratio/max": 1.2594510316848755,
-      "sampling/importance_sampling_ratio/mean": 0.8206984996795654,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 2.6992740631103516,
-      "sampling/sampling_logp_difference/mean": 0.07161708176136017,
-      "step": 20,
-      "step_time": 11.206723564000072
-    },
-    {
-      "clip_ratio/high_max": 0.02500000037252903,
-      "clip_ratio/high_mean": 0.012500000186264515,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.012500000186264515,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 372.53125,
-      "completions/mean_terminated_length": 372.53125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.057463863951852545,
-      "epoch": 0.00042,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 3.4608774185180664,
-      "kl": 1.832482981204521,
-      "learning_rate": 5.714285714285714e-05,
-      "loss": -0.019,
-      "num_tokens": 1129382.0,
-      "reward": -0.26816296577453613,
-      "reward_std": 0.8223587274551392,
-      "rewards/rollout_reward_func/mean": -0.26816296577453613,
-      "rewards/rollout_reward_func/std": 0.8097085952758789,
-      "sampling/importance_sampling_ratio/max": 1.1088007688522339,
-      "sampling/importance_sampling_ratio/mean": 0.9074119329452515,
-      "sampling/importance_sampling_ratio/min": 0.05796723812818527,
-      "sampling/sampling_logp_difference/max": 2.9442572593688965,
-      "sampling/sampling_logp_difference/mean": 0.06413605809211731,
-      "step": 21,
-      "step_time": 13.82654130900005
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.010416666977107525,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 296.90625,
-      "completions/mean_terminated_length": 296.90625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.06721748731797561,
-      "epoch": 0.00044,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.4418312311172485,
-      "kl": 2.94984346779529,
-      "learning_rate": 6e-05,
-      "loss": -0.0289,
-      "num_tokens": 1184372.0,
-      "reward": -0.24570472538471222,
-      "reward_std": 0.9054995775222778,
-      "rewards/rollout_reward_func/mean": -0.24570472538471222,
-      "rewards/rollout_reward_func/std": 0.9906948804855347,
-      "sampling/importance_sampling_ratio/max": 1.7179746627807617,
-      "sampling/importance_sampling_ratio/mean": 0.9893367290496826,
-      "sampling/importance_sampling_ratio/min": 0.19051571190357208,
-      "sampling/sampling_logp_difference/max": 2.113297462463379,
-      "sampling/sampling_logp_difference/mean": 0.04138573259115219,
-      "step": 22,
-      "step_time": 12.665812612000082
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0052083334885537624,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0052083334885537624,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 677.0,
-      "completions/max_terminated_length": 677.0,
-      "completions/mean_length": 368.3125,
-      "completions/mean_terminated_length": 368.3125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.06680563138797879,
-      "epoch": 0.00046,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.0064221620559692,
-      "kl": 2.083964512392413,
-      "learning_rate": 6.285714285714286e-05,
-      "loss": 0.0211,
-      "num_tokens": 1242719.0,
-      "reward": 0.005538810044527054,
-      "reward_std": 0.9316948652267456,
-      "rewards/rollout_reward_func/mean": 0.005538810044527054,
-      "rewards/rollout_reward_func/std": 0.9904869198799133,
-      "sampling/importance_sampling_ratio/max": 2.143568277359009,
-      "sampling/importance_sampling_ratio/mean": 1.0061091184616089,
-      "sampling/importance_sampling_ratio/min": 0.2605152428150177,
-      "sampling/sampling_logp_difference/max": 1.3374133110046387,
-      "sampling/sampling_logp_difference/mean": 0.03522632271051407,
-      "step": 23,
-      "step_time": 13.648552884999958
-    },
-    {
-      "clip_ratio/high_max": 0.0069444444961845875,
-      "clip_ratio/high_mean": 0.0034722222480922937,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.013888889225199819,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 274.1875,
-      "completions/mean_terminated_length": 274.1875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.0993105829693377,
-      "epoch": 0.00048,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.5184568166732788,
-      "kl": 1.1476193765411153,
-      "learning_rate": 6.571428571428571e-05,
-      "loss": -0.0219,
-      "num_tokens": 1298050.0,
-      "reward": -0.2780725955963135,
-      "reward_std": 0.760818362236023,
-      "rewards/rollout_reward_func/mean": -0.2780725955963135,
-      "rewards/rollout_reward_func/std": 0.9655778408050537,
-      "sampling/importance_sampling_ratio/max": 1.1772704124450684,
-      "sampling/importance_sampling_ratio/mean": 0.9638389348983765,
-      "sampling/importance_sampling_ratio/min": 0.14113064110279083,
-      "sampling/sampling_logp_difference/max": 1.9840805530548096,
-      "sampling/sampling_logp_difference/mean": 0.026738665997982025,
-      "step": 24,
-      "step_time": 11.18192802700014
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.0024038462433964014,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.008653846336528659,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 722.0,
-      "completions/max_terminated_length": 722.0,
-      "completions/mean_length": 413.90625,
-      "completions/mean_terminated_length": 413.90625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.17944650270510465,
-      "epoch": 0.0005,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.7797176837921143,
-      "kl": 1.3149332464672625,
-      "learning_rate": 6.857142857142858e-05,
-      "loss": -0.0168,
-      "num_tokens": 1360038.0,
-      "reward": -0.10542695969343185,
-      "reward_std": 0.8592759370803833,
-      "rewards/rollout_reward_func/mean": -0.10542695969343185,
-      "rewards/rollout_reward_func/std": 0.9378611445426941,
-      "sampling/importance_sampling_ratio/max": 1.282301664352417,
-      "sampling/importance_sampling_ratio/mean": 0.9609808921813965,
-      "sampling/importance_sampling_ratio/min": 0.13269297778606415,
-      "sampling/sampling_logp_difference/max": 2.019876003265381,
-      "sampling/sampling_logp_difference/mean": 0.03994935378432274,
-      "step": 25,
-      "step_time": 13.790912540000136
-    },
-    {
-      "clip_ratio/high_max": 0.037500000558793545,
-      "clip_ratio/high_mean": 0.018750000279396772,
-      "clip_ratio/low_mean": 0.0034722222480922937,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.022222222527489066,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 361.125,
-      "completions/mean_terminated_length": 361.125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.22724986402317882,
-      "epoch": 0.00052,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.9671414494514465,
-      "kl": 1.0795668102800846,
-      "learning_rate": 7.142857142857143e-05,
-      "loss": 0.0021,
-      "num_tokens": 1415399.0,
-      "reward": -0.1481989622116089,
-      "reward_std": 0.9951958656311035,
-      "rewards/rollout_reward_func/mean": -0.1481989622116089,
-      "rewards/rollout_reward_func/std": 0.9867257475852966,
-      "sampling/importance_sampling_ratio/max": 1.2668951749801636,
-      "sampling/importance_sampling_ratio/mean": 0.891692042350769,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 2.270904779434204,
-      "sampling/sampling_logp_difference/mean": 0.05143989250063896,
-      "step": 26,
-      "step_time": 12.553581390999852
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.010416666977107525,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 303.1875,
-      "completions/mean_terminated_length": 303.1875,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.2595046339556575,
-      "epoch": 0.00054,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.5745887160301208,
-      "kl": 1.365425143390894,
-      "learning_rate": 7.428571428571429e-05,
-      "loss": -0.0369,
-      "num_tokens": 1470660.0,
-      "reward": -0.13661298155784607,
-      "reward_std": 0.8571063876152039,
-      "rewards/rollout_reward_func/mean": -0.13661298155784607,
-      "rewards/rollout_reward_func/std": 0.8741553425788879,
-      "sampling/importance_sampling_ratio/max": 1.4167910814285278,
-      "sampling/importance_sampling_ratio/mean": 0.9384365081787109,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 1.243765115737915,
-      "sampling/sampling_logp_difference/mean": 0.04462525248527527,
-      "step": 27,
-      "step_time": 13.429217179000261
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.0052083334885537624,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.01145833358168602,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 301.34375,
-      "completions/mean_terminated_length": 301.34375,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.2221351945772767,
-      "epoch": 0.00056,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.4858068227767944,
-      "kl": 1.6639087870717049,
-      "learning_rate": 7.714285714285715e-05,
-      "loss": -0.0743,
-      "num_tokens": 1524604.0,
-      "reward": 0.15634837746620178,
-      "reward_std": 0.8471760749816895,
-      "rewards/rollout_reward_func/mean": 0.15634837746620178,
-      "rewards/rollout_reward_func/std": 0.8385409712791443,
-      "sampling/importance_sampling_ratio/max": 2.691802501678467,
-      "sampling/importance_sampling_ratio/mean": 1.079996109008789,
-      "sampling/importance_sampling_ratio/min": 0.5096427798271179,
-      "sampling/sampling_logp_difference/max": 0.5276076793670654,
-      "sampling/sampling_logp_difference/mean": 0.0421307310461998,
-      "step": 28,
-      "step_time": 11.09212382000021
-    },
-    {
-      "clip_ratio/high_max": 0.034722222946584225,
-      "clip_ratio/high_mean": 0.017361111473292112,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.027777778450399637,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 345.75,
-      "completions/mean_terminated_length": 345.75,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.20573155442252755,
-      "epoch": 0.00058,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.5787720084190369,
-      "kl": 1.9223689641803503,
-      "learning_rate": 8e-05,
-      "loss": -0.0152,
-      "num_tokens": 1583295.0,
-      "reward": 0.220237597823143,
-      "reward_std": 0.7271397113800049,
-      "rewards/rollout_reward_func/mean": 0.220237597823143,
-      "rewards/rollout_reward_func/std": 0.7316510677337646,
-      "sampling/importance_sampling_ratio/max": 1.1069331169128418,
-      "sampling/importance_sampling_ratio/mean": 0.9354293346405029,
-      "sampling/importance_sampling_ratio/min": 0.3376814126968384,
-      "sampling/sampling_logp_difference/max": 0.8825469017028809,
-      "sampling/sampling_logp_difference/mean": 0.03632263094186783,
-      "step": 29,
-      "step_time": 12.292719963999843
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0062500000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 342.3125,
-      "completions/mean_terminated_length": 342.3125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.06238462869077921,
-      "epoch": 0.0006,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.47024017572402954,
-      "kl": 2.624549888074398,
-      "learning_rate": 8.285714285714287e-05,
-      "loss": 0.0209,
-      "num_tokens": 1637771.0,
-      "reward": 0.548281192779541,
-      "reward_std": 0.45720434188842773,
-      "rewards/rollout_reward_func/mean": 0.548281192779541,
-      "rewards/rollout_reward_func/std": 0.5005651712417603,
-      "sampling/importance_sampling_ratio/max": 1.4442425966262817,
-      "sampling/importance_sampling_ratio/mean": 1.0311418771743774,
-      "sampling/importance_sampling_ratio/min": 0.8194373250007629,
-      "sampling/sampling_logp_difference/max": 0.5042067766189575,
-      "sampling/sampling_logp_difference/mean": 0.013192035257816315,
-      "step": 30,
-      "step_time": 10.964770865000105
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 713.0,
-      "completions/max_terminated_length": 713.0,
-      "completions/mean_length": 269.6875,
-      "completions/mean_terminated_length": 269.6875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.0758107526198728,
-      "epoch": 0.00062,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.25941506028175354,
-      "kl": 2.267256945371628,
-      "learning_rate": 8.571428571428571e-05,
-      "loss": 0.0256,
-      "num_tokens": 1694542.0,
-      "reward": 0.5457811951637268,
-      "reward_std": 0.4714530110359192,
-      "rewards/rollout_reward_func/mean": 0.5457811951637268,
-      "rewards/rollout_reward_func/std": 0.5039091110229492,
-      "sampling/importance_sampling_ratio/max": 1.4384585618972778,
-      "sampling/importance_sampling_ratio/mean": 1.0167803764343262,
-      "sampling/importance_sampling_ratio/min": 0.8918675184249878,
-      "sampling/sampling_logp_difference/max": 0.3632171154022217,
-      "sampling/sampling_logp_difference/mean": 0.007435648236423731,
-      "step": 31,
-      "step_time": 11.205504815999916
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 279.625,
-      "completions/mean_terminated_length": 279.625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.05832514737267047,
-      "epoch": 0.00064,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.17318402230739594,
-      "kl": 2.250509649515152,
-      "learning_rate": 8.857142857142857e-05,
-      "loss": 0.0107,
-      "num_tokens": 1749731.0,
-      "reward": 0.4203124940395355,
-      "reward_std": 0.5108482837677002,
-      "rewards/rollout_reward_func/mean": 0.4203124940395355,
-      "rewards/rollout_reward_func/std": 0.497097909450531,
-      "sampling/importance_sampling_ratio/max": 1.1264947652816772,
-      "sampling/importance_sampling_ratio/mean": 0.9992133378982544,
-      "sampling/importance_sampling_ratio/min": 0.8888209462165833,
-      "sampling/sampling_logp_difference/max": 0.11888670921325684,
-      "sampling/sampling_logp_difference/mean": 0.004699671640992165,
-      "step": 32,
-      "step_time": 11.379408340000282
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 722.0,
-      "completions/max_terminated_length": 722.0,
-      "completions/mean_length": 166.5625,
-      "completions/mean_terminated_length": 166.5625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.11914130803779699,
-      "epoch": 0.00066,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.35149407386779785,
-      "kl": 2.117000125348568,
-      "learning_rate": 9.142857142857143e-05,
-      "loss": -0.0073,
-      "num_tokens": 1800761.0,
-      "reward": 0.4790624976158142,
-      "reward_std": 0.4966807961463928,
-      "rewards/rollout_reward_func/mean": 0.4790624976158142,
-      "rewards/rollout_reward_func/std": 0.5088212490081787,
-      "sampling/importance_sampling_ratio/max": 1.0266004800796509,
-      "sampling/importance_sampling_ratio/mean": 0.9677810668945312,
-      "sampling/importance_sampling_ratio/min": 0.6308969855308533,
-      "sampling/sampling_logp_difference/max": 0.46265050768852234,
-      "sampling/sampling_logp_difference/mean": 0.01113891787827015,
-      "step": 33,
-      "step_time": 11.08241834699993
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0062500000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 300.84375,
-      "completions/mean_terminated_length": 300.84375,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.184289012933732,
-      "epoch": 0.00068,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.4543190002441406,
-      "kl": 2.805480755865574,
-      "learning_rate": 9.428571428571429e-05,
-      "loss": 0.0115,
-      "num_tokens": 1857463.0,
-      "reward": 0.6110936999320984,
-      "reward_std": 0.5366275906562805,
-      "rewards/rollout_reward_func/mean": 0.6110936999320984,
-      "rewards/rollout_reward_func/std": 0.557543158531189,
-      "sampling/importance_sampling_ratio/max": 1.9884446859359741,
-      "sampling/importance_sampling_ratio/mean": 1.0303646326065063,
-      "sampling/importance_sampling_ratio/min": 0.5916227102279663,
-      "sampling/sampling_logp_difference/max": 0.7394986152648926,
-      "sampling/sampling_logp_difference/mean": 0.03770880401134491,
-      "step": 34,
-      "step_time": 11.234121857000673
-    },
-    {
-      "clip_ratio/high_max": 0.0416666679084301,
-      "clip_ratio/high_mean": 0.02083333395421505,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.031250000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 695.0,
-      "completions/max_terminated_length": 695.0,
-      "completions/mean_length": 264.875,
-      "completions/mean_terminated_length": 264.875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.18178106704726815,
-      "epoch": 0.0007,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.44406557083129883,
-      "kl": 3.126620076596737,
-      "learning_rate": 9.714285714285715e-05,
-      "loss": -0.0033,
-      "num_tokens": 1913660.0,
-      "reward": 0.5779687166213989,
-      "reward_std": 0.5265768766403198,
-      "rewards/rollout_reward_func/mean": 0.5779687166213989,
-      "rewards/rollout_reward_func/std": 0.5046586990356445,
-      "sampling/importance_sampling_ratio/max": 1.506946086883545,
-      "sampling/importance_sampling_ratio/mean": 0.8585790395736694,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 2.0860564708709717,
-      "sampling/sampling_logp_difference/mean": 0.07170744985342026,
-      "step": 35,
-      "step_time": 11.274823958999377
-    },
-    {
-      "clip_ratio/high_max": 0.02500000037252903,
-      "clip_ratio/high_mean": 0.012500000186264515,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.02291666716337204,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 668.0,
-      "completions/max_terminated_length": 668.0,
-      "completions/mean_length": 207.90625,
-      "completions/mean_terminated_length": 207.90625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.19994325749576092,
-      "epoch": 0.00072,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.8594412207603455,
-      "kl": 3.070635214447975,
-      "learning_rate": 0.0001,
-      "loss": 0.0217,
-      "num_tokens": 1962669.0,
-      "reward": 0.7246874570846558,
-      "reward_std": 0.3917328715324402,
-      "rewards/rollout_reward_func/mean": 0.7246874570846558,
-      "rewards/rollout_reward_func/std": 0.4577009975910187,
-      "sampling/importance_sampling_ratio/max": 2.865489959716797,
-      "sampling/importance_sampling_ratio/mean": 0.9528592228889465,
-      "sampling/importance_sampling_ratio/min": 0.20568400621414185,
-      "sampling/sampling_logp_difference/max": 1.5809779167175293,
-      "sampling/sampling_logp_difference/mean": 0.1094856858253479,
-      "step": 36,
-      "step_time": 11.00211126499994
-    },
-    {
-      "clip_ratio/high_max": 0.010416666977107525,
-      "clip_ratio/high_mean": 0.0052083334885537624,
-      "clip_ratio/low_mean": 0.012500000186264515,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.017708333674818277,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 325.40625,
-      "completions/mean_terminated_length": 325.40625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.09332124679349363,
-      "epoch": 0.00074,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.694848895072937,
-      "kl": 9.272021323442459,
-      "learning_rate": 9.999994059657532e-05,
-      "loss": 0.0465,
-      "num_tokens": 2017569.0,
-      "reward": 0.48249998688697815,
-      "reward_std": 0.5203092098236084,
-      "rewards/rollout_reward_func/mean": 0.48249998688697815,
-      "rewards/rollout_reward_func/std": 0.5045086741447449,
-      "sampling/importance_sampling_ratio/max": 1.3940647840499878,
-      "sampling/importance_sampling_ratio/mean": 0.9123125076293945,
-      "sampling/importance_sampling_ratio/min": 0.24273724853992462,
-      "sampling/sampling_logp_difference/max": 1.4541773796081543,
-      "sampling/sampling_logp_difference/mean": 0.06990130990743637,
-      "step": 37,
-      "step_time": 11.3070248490003
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.016666667070239782,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 343.78125,
-      "completions/mean_terminated_length": 343.78125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.1606664622668177,
-      "epoch": 0.00076,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.0721571445465088,
-      "kl": 4.78829974681139,
-      "learning_rate": 9.999976238648947e-05,
-      "loss": 0.0325,
-      "num_tokens": 2074090.0,
-      "reward": 0.7437499761581421,
-      "reward_std": 0.43693897128105164,
-      "rewards/rollout_reward_func/mean": 0.7437499761581421,
-      "rewards/rollout_reward_func/std": 0.45396992564201355,
-      "sampling/importance_sampling_ratio/max": 1.6197967529296875,
-      "sampling/importance_sampling_ratio/mean": 0.8961669206619263,
-      "sampling/importance_sampling_ratio/min": 0.2103896141052246,
-      "sampling/sampling_logp_difference/max": 1.5587217807769775,
-      "sampling/sampling_logp_difference/mean": 0.08295002579689026,
-      "step": 38,
-      "step_time": 11.355166078999673
-    },
-    {
-      "clip_ratio/high_max": 0.033333334140479565,
-      "clip_ratio/high_mean": 0.016666667070239782,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.016666667070239782,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 362.78125,
-      "completions/mean_terminated_length": 362.78125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.2447179276496172,
-      "epoch": 0.00078,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.9297128319740295,
-      "kl": 4.717592045664787,
-      "learning_rate": 9.999946537030704e-05,
-      "loss": 0.0091,
-      "num_tokens": 2134460.0,
-      "reward": 0.4856249988079071,
-      "reward_std": 0.5640056133270264,
-      "rewards/rollout_reward_func/mean": 0.4856249988079071,
-      "rewards/rollout_reward_func/std": 0.5751574039459229,
-      "sampling/importance_sampling_ratio/max": 1.2337568998336792,
-      "sampling/importance_sampling_ratio/mean": 0.8471465110778809,
-      "sampling/importance_sampling_ratio/min": 0.44463473558425903,
-      "sampling/sampling_logp_difference/max": 0.8104925155639648,
-      "sampling/sampling_logp_difference/mean": 0.05927025154232979,
-      "step": 39,
-      "step_time": 10.88231969100002
-    },
-    {
-      "clip_ratio/high_max": 0.03194444486871362,
-      "clip_ratio/high_mean": 0.01597222243435681,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.01597222243435681,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 677.0,
-      "completions/max_terminated_length": 677.0,
-      "completions/mean_length": 210.71875,
-      "completions/mean_terminated_length": 210.71875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.1360793736530468,
-      "epoch": 0.0008,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.29666247963905334,
-      "kl": 6.856255024671555,
-      "learning_rate": 9.999904954896906e-05,
-      "loss": 0.0139,
-      "num_tokens": 2184568.0,
-      "reward": 0.6371874809265137,
-      "reward_std": 0.36927926540374756,
-      "rewards/rollout_reward_func/mean": 0.6371874809265137,
-      "rewards/rollout_reward_func/std": 0.49185124039649963,
-      "sampling/importance_sampling_ratio/max": 1.198319435119629,
-      "sampling/importance_sampling_ratio/mean": 0.9105734825134277,
-      "sampling/importance_sampling_ratio/min": 0.38900867104530334,
-      "sampling/sampling_logp_difference/max": 0.9301635026931763,
-      "sampling/sampling_logp_difference/mean": 0.048708051443099976,
-      "step": 40,
-      "step_time": 11.448123117999785
-    },
-    {
-      "clip_ratio/high_max": 0.02500000037252903,
-      "clip_ratio/high_mean": 0.012500000186264515,
-      "clip_ratio/low_mean": 0.0062500000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.018750000279396772,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 236.90625,
-      "completions/mean_terminated_length": 236.90625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.09502423234516755,
-      "epoch": 0.00082,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.45557841658592224,
-      "kl": 7.105475306510925,
-      "learning_rate": 9.999851492379291e-05,
-      "loss": 0.0155,
-      "num_tokens": 2238067.0,
-      "reward": 0.5724999904632568,
-      "reward_std": 0.5083093643188477,
-      "rewards/rollout_reward_func/mean": 0.5724999904632568,
-      "rewards/rollout_reward_func/std": 0.5038048624992371,
-      "sampling/importance_sampling_ratio/max": 1.8395185470581055,
-      "sampling/importance_sampling_ratio/mean": 0.9868057370185852,
-      "sampling/importance_sampling_ratio/min": 0.3422168493270874,
-      "sampling/sampling_logp_difference/max": 1.426053762435913,
-      "sampling/sampling_logp_difference/mean": 0.04565257206559181,
-      "step": 41,
-      "step_time": 10.813364009000452
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 722.0,
-      "completions/max_terminated_length": 722.0,
-      "completions/mean_length": 185.375,
-      "completions/mean_terminated_length": 185.375,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.06905311258742586,
-      "epoch": 0.00084,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.5812671184539795,
-      "kl": 6.845378398895264,
-      "learning_rate": 9.999786149647238e-05,
-      "loss": 0.0047,
-      "num_tokens": 2289758.0,
-      "reward": 0.5425000190734863,
-      "reward_std": 0.5230197906494141,
-      "rewards/rollout_reward_func/mean": 0.5425000190734863,
-      "rewards/rollout_reward_func/std": 0.5084131360054016,
-      "sampling/importance_sampling_ratio/max": 2.039076566696167,
-      "sampling/importance_sampling_ratio/mean": 1.0192711353302002,
-      "sampling/importance_sampling_ratio/min": 0.4893311560153961,
-      "sampling/sampling_logp_difference/max": 0.7137637138366699,
-      "sampling/sampling_logp_difference/mean": 0.02361428365111351,
-      "step": 42,
-      "step_time": 11.764213820000123
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 686.0,
-      "completions/max_terminated_length": 686.0,
-      "completions/mean_length": 179.8125,
-      "completions/mean_terminated_length": 179.8125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.05975941789802164,
-      "epoch": 0.00086,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2592904269695282,
-      "kl": 7.11406484246254,
-      "learning_rate": 9.999708926907767e-05,
-      "loss": 0.0215,
-      "num_tokens": 2341040.0,
-      "reward": 0.5406249761581421,
-      "reward_std": 0.5185223817825317,
-      "rewards/rollout_reward_func/mean": 0.5406249761581421,
-      "rewards/rollout_reward_func/std": 0.5046936869621277,
-      "sampling/importance_sampling_ratio/max": 1.2536290884017944,
-      "sampling/importance_sampling_ratio/mean": 0.9912163019180298,
-      "sampling/importance_sampling_ratio/min": 0.5534382462501526,
-      "sampling/sampling_logp_difference/max": 0.5880619287490845,
-      "sampling/sampling_logp_difference/mean": 0.009908081963658333,
-      "step": 43,
-      "step_time": 10.66558726900098
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.010416666977107525,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 286.78125,
-      "completions/mean_terminated_length": 286.78125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.08994953951332718,
-      "epoch": 0.00088,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.3893328309059143,
-      "kl": 5.922937855124474,
-      "learning_rate": 9.99961982440553e-05,
-      "loss": -0.0091,
-      "num_tokens": 2396978.0,
-      "reward": 0.48009711503982544,
-      "reward_std": 0.5554372072219849,
-      "rewards/rollout_reward_func/mean": 0.48009711503982544,
-      "rewards/rollout_reward_func/std": 0.582542359828949,
-      "sampling/importance_sampling_ratio/max": 1.8523739576339722,
-      "sampling/importance_sampling_ratio/mean": 1.0359950065612793,
-      "sampling/importance_sampling_ratio/min": 0.6423361897468567,
-      "sampling/sampling_logp_difference/max": 0.6209536790847778,
-      "sampling/sampling_logp_difference/mean": 0.02452177368104458,
-      "step": 44,
-      "step_time": 11.885553534999872
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 310.53125,
-      "completions/mean_terminated_length": 310.53125,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.12320263730362058,
-      "epoch": 0.0009,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2887427508831024,
-      "kl": 5.701345443725586,
-      "learning_rate": 9.999518842422821e-05,
-      "loss": 0.0152,
-      "num_tokens": 2453615.0,
-      "reward": 0.5428124666213989,
-      "reward_std": 0.5223480463027954,
-      "rewards/rollout_reward_func/mean": 0.5428124666213989,
-      "rewards/rollout_reward_func/std": 0.5661541819572449,
-      "sampling/importance_sampling_ratio/max": 1.2400528192520142,
-      "sampling/importance_sampling_ratio/mean": 0.9792937636375427,
-      "sampling/importance_sampling_ratio/min": 0.5575809478759766,
-      "sampling/sampling_logp_difference/max": 0.6647803783416748,
-      "sampling/sampling_logp_difference/mean": 0.027244437485933304,
-      "step": 45,
-      "step_time": 11.027731855000411
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0062500000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 267.875,
-      "completions/mean_terminated_length": 267.875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.07362117519369349,
-      "epoch": 0.00092,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.3012773096561432,
-      "kl": 6.29687874019146,
-      "learning_rate": 9.999405981279571e-05,
-      "loss": 0.0437,
-      "num_tokens": 2508844.0,
-      "reward": 0.4506250023841858,
-      "reward_std": 0.4844134747982025,
-      "rewards/rollout_reward_func/mean": 0.4506250023841858,
-      "rewards/rollout_reward_func/std": 0.49820929765701294,
-      "sampling/importance_sampling_ratio/max": 1.8740555047988892,
-      "sampling/importance_sampling_ratio/mean": 0.9862356185913086,
-      "sampling/importance_sampling_ratio/min": 0.2873587906360626,
-      "sampling/sampling_logp_difference/max": 1.2469677925109863,
-      "sampling/sampling_logp_difference/mean": 0.034819282591342926,
-      "step": 46,
-      "step_time": 11.991214588000503
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 713.0,
-      "completions/max_terminated_length": 713.0,
-      "completions/mean_length": 194.03125,
-      "completions/mean_terminated_length": 194.03125,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.07023912924341857,
-      "epoch": 0.00094,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2493385225534439,
-      "kl": 8.241001963615417,
-      "learning_rate": 9.999281241333345e-05,
-      "loss": -0.0092,
-      "num_tokens": 2559132.0,
-      "reward": 0.38718748092651367,
-      "reward_std": 0.6814954280853271,
-      "rewards/rollout_reward_func/mean": 0.38718748092651367,
-      "rewards/rollout_reward_func/std": 0.6587866544723511,
-      "sampling/importance_sampling_ratio/max": 1.0057464838027954,
-      "sampling/importance_sampling_ratio/mean": 0.9433271884918213,
-      "sampling/importance_sampling_ratio/min": 0.6084576845169067,
-      "sampling/sampling_logp_difference/max": 0.506122350692749,
-      "sampling/sampling_logp_difference/mean": 0.01987236924469471,
-      "step": 47,
-      "step_time": 10.563781170000766
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 188.25,
-      "completions/mean_terminated_length": 188.25,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.060785901034250855,
-      "epoch": 0.00096,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2123745232820511,
-      "kl": 6.653421573340893,
-      "learning_rate": 9.999144622979339e-05,
-      "loss": 0.0036,
-      "num_tokens": 2613947.0,
-      "reward": 0.4478124976158142,
-      "reward_std": 0.5109598636627197,
-      "rewards/rollout_reward_func/mean": 0.4478124976158142,
-      "rewards/rollout_reward_func/std": 0.5064979791641235,
-      "sampling/importance_sampling_ratio/max": 1.33092200756073,
-      "sampling/importance_sampling_ratio/mean": 0.9957506656646729,
-      "sampling/importance_sampling_ratio/min": 0.6696221828460693,
-      "sampling/sampling_logp_difference/max": 0.3861722946166992,
-      "sampling/sampling_logp_difference/mean": 0.011901838704943657,
-      "step": 48,
-      "step_time": 12.05637606399955
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 713.0,
-      "completions/max_terminated_length": 713.0,
-      "completions/mean_length": 265.28125,
-      "completions/mean_terminated_length": 265.28125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.07921588758472353,
-      "epoch": 0.00098,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.3492256700992584,
-      "kl": 5.599457576870918,
-      "learning_rate": 9.998996126650389e-05,
-      "loss": 0.0033,
-      "num_tokens": 2665835.0,
-      "reward": 0.5462499856948853,
-      "reward_std": 0.5150694847106934,
-      "rewards/rollout_reward_func/mean": 0.5462499856948853,
-      "rewards/rollout_reward_func/std": 0.50818932056427,
-      "sampling/importance_sampling_ratio/max": 1.246306300163269,
-      "sampling/importance_sampling_ratio/mean": 0.9999866485595703,
-      "sampling/importance_sampling_ratio/min": 0.8210906982421875,
-      "sampling/sampling_logp_difference/max": 0.2230236530303955,
-      "sampling/sampling_logp_difference/mean": 0.008543935604393482,
-      "step": 49,
-      "step_time": 10.708852489000265
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0062500000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 686.0,
-      "completions/max_terminated_length": 686.0,
-      "completions/mean_length": 291.3125,
-      "completions/mean_terminated_length": 291.3125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.06653009611181915,
-      "epoch": 0.001,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18226270377635956,
-      "kl": 5.763965398073196,
-      "learning_rate": 9.998835752816952e-05,
-      "loss": -0.0024,
-      "num_tokens": 2724829.0,
-      "reward": 0.6106250286102295,
-      "reward_std": 0.4943864345550537,
-      "rewards/rollout_reward_func/mean": 0.6106250286102295,
-      "rewards/rollout_reward_func/std": 0.5036827921867371,
-      "sampling/importance_sampling_ratio/max": 1.9964224100112915,
-      "sampling/importance_sampling_ratio/mean": 1.0471532344818115,
-      "sampling/importance_sampling_ratio/min": 0.8690345883369446,
-      "sampling/sampling_logp_difference/max": 0.691453218460083,
-      "sampling/sampling_logp_difference/mean": 0.015394123271107674,
-      "step": 50,
-      "step_time": 11.142331651999484
-    },
-    {
-      "clip_ratio/high_max": 0.02083333395421505,
-      "clip_ratio/high_mean": 0.010416666977107525,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.010416666977107525,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 200.625,
-      "completions/mean_terminated_length": 200.625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.08440758194774389,
-      "epoch": 0.00102,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.007480502128601,
-      "kl": 7.743502289056778,
-      "learning_rate": 9.99866350198713e-05,
-      "loss": -0.0003,
-      "num_tokens": 2777271.0,
-      "reward": 0.5743749737739563,
-      "reward_std": 0.5210629105567932,
-      "rewards/rollout_reward_func/mean": 0.5743749737739563,
-      "rewards/rollout_reward_func/std": 0.5072184205055237,
-      "sampling/importance_sampling_ratio/max": 1.346501111984253,
-      "sampling/importance_sampling_ratio/mean": 0.975390613079071,
-      "sampling/importance_sampling_ratio/min": 0.48252570629119873,
-      "sampling/sampling_logp_difference/max": 0.7286257743835449,
-      "sampling/sampling_logp_difference/mean": 0.01844225823879242,
-      "step": 51,
-      "step_time": 11.411072066000315
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 267.15625,
-      "completions/mean_terminated_length": 267.15625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.08491186366882175,
-      "epoch": 0.00104,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.15097106993198395,
-      "kl": 5.030089437961578,
-      "learning_rate": 9.99847937470664e-05,
-      "loss": 0.0086,
-      "num_tokens": 2833648.0,
-      "reward": 0.7043749690055847,
-      "reward_std": 0.46712028980255127,
-      "rewards/rollout_reward_func/mean": 0.7043749690055847,
-      "rewards/rollout_reward_func/std": 0.4704008996486664,
-      "sampling/importance_sampling_ratio/max": 1.2003308534622192,
-      "sampling/importance_sampling_ratio/mean": 1.0036582946777344,
-      "sampling/importance_sampling_ratio/min": 0.9710050225257874,
-      "sampling/sampling_logp_difference/max": 0.18256908655166626,
-      "sampling/sampling_logp_difference/mean": 0.0034763971343636513,
-      "step": 52,
-      "step_time": 11.31483540899967
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 739.0,
-      "completions/max_terminated_length": 739.0,
-      "completions/mean_length": 245.53125,
-      "completions/mean_terminated_length": 245.53125,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.2502337893238291,
-      "epoch": 0.00106,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.0586179494857788,
-      "kl": 5.173554673790932,
-      "learning_rate": 9.99828337155883e-05,
-      "loss": 0.0167,
-      "num_tokens": 2887869.0,
-      "reward": 0.504687488079071,
-      "reward_std": 0.5709031224250793,
-      "rewards/rollout_reward_func/mean": 0.504687488079071,
-      "rewards/rollout_reward_func/std": 0.570754885673523,
-      "sampling/importance_sampling_ratio/max": 1.9516154527664185,
-      "sampling/importance_sampling_ratio/mean": 1.068804144859314,
-      "sampling/importance_sampling_ratio/min": 0.9312329292297363,
-      "sampling/sampling_logp_difference/max": 0.6652579307556152,
-      "sampling/sampling_logp_difference/mean": 0.030817098915576935,
-      "step": 53,
-      "step_time": 11.169498285998998
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0078125,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0078125,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 273.5,
-      "completions/mean_terminated_length": 273.5,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.2743898332118988,
-      "epoch": 0.00108,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.9803305268287659,
-      "kl": 4.051570408046246,
-      "learning_rate": 9.998075493164677e-05,
-      "loss": 0.0176,
-      "num_tokens": 2939911.0,
-      "reward": 0.5649999380111694,
-      "reward_std": 0.567491888999939,
-      "rewards/rollout_reward_func/mean": 0.5649999380111694,
-      "rewards/rollout_reward_func/std": 0.5686146020889282,
-      "sampling/importance_sampling_ratio/max": 2.429133176803589,
-      "sampling/importance_sampling_ratio/mean": 1.0337567329406738,
-      "sampling/importance_sampling_ratio/min": 0.6068115234375,
-      "sampling/sampling_logp_difference/max": 0.8871572017669678,
-      "sampling/sampling_logp_difference/mean": 0.03650107979774475,
-      "step": 54,
-      "step_time": 10.936946854000098
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 308.21875,
-      "completions/mean_terminated_length": 308.21875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.12360220204573125,
-      "epoch": 0.0011,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.3061436414718628,
-      "kl": 2.5437240228056908,
-      "learning_rate": 9.997855740182778e-05,
-      "loss": -0.0067,
-      "num_tokens": 2997274.0,
-      "reward": 0.5760937333106995,
-      "reward_std": 0.4665103554725647,
-      "rewards/rollout_reward_func/mean": 0.5760937333106995,
-      "rewards/rollout_reward_func/std": 0.4992554485797882,
-      "sampling/importance_sampling_ratio/max": 1.0957883596420288,
-      "sampling/importance_sampling_ratio/mean": 0.982556939125061,
-      "sampling/importance_sampling_ratio/min": 0.6691784262657166,
-      "sampling/sampling_logp_difference/max": 0.4017000198364258,
-      "sampling/sampling_logp_difference/mean": 0.012154795229434967,
-      "step": 55,
-      "step_time": 12.132875299000716
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 713.0,
-      "completions/max_terminated_length": 713.0,
-      "completions/mean_length": 251.1875,
-      "completions/mean_terminated_length": 251.1875,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.08530077605973929,
-      "epoch": 0.00112,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.4645513892173767,
-      "kl": 2.6704555079340935,
-      "learning_rate": 9.997624113309345e-05,
-      "loss": 0.0079,
-      "num_tokens": 3054055.0,
-      "reward": 0.5759375095367432,
-      "reward_std": 0.5824880599975586,
-      "rewards/rollout_reward_func/mean": 0.5759375095367432,
-      "rewards/rollout_reward_func/std": 0.564611554145813,
-      "sampling/importance_sampling_ratio/max": 1.49051833152771,
-      "sampling/importance_sampling_ratio/mean": 1.005382776260376,
-      "sampling/importance_sampling_ratio/min": 0.8475367426872253,
-      "sampling/sampling_logp_difference/max": 0.40567874908447266,
-      "sampling/sampling_logp_difference/mean": 0.006726027466356754,
-      "step": 56,
-      "step_time": 10.839855434000356
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 340.53125,
-      "completions/mean_terminated_length": 340.53125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.03184928291011602,
-      "epoch": 0.00114,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.17147044837474823,
-      "kl": 3.311452843248844,
-      "learning_rate": 9.99738061327822e-05,
-      "loss": -0.0023,
-      "num_tokens": 3113860.0,
-      "reward": 0.5509374737739563,
-      "reward_std": 0.5238043069839478,
-      "rewards/rollout_reward_func/mean": 0.5509374737739563,
-      "rewards/rollout_reward_func/std": 0.5088212490081787,
-      "sampling/importance_sampling_ratio/max": 1.012040376663208,
-      "sampling/importance_sampling_ratio/mean": 0.9918051958084106,
-      "sampling/importance_sampling_ratio/min": 0.6893038749694824,
-      "sampling/sampling_logp_difference/max": 0.3707852363586426,
-      "sampling/sampling_logp_difference/mean": 0.0032549004536122084,
-      "step": 57,
-      "step_time": 11.785106229999656
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.010416666977107525,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.010416666977107525,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 235.78125,
-      "completions/mean_terminated_length": 235.78125,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.06814532617863733,
-      "epoch": 0.00116,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.4730847477912903,
-      "kl": 2.127237744629383,
-      "learning_rate": 9.997125240860854e-05,
-      "loss": 0.0033,
-      "num_tokens": 3168830.0,
-      "reward": 0.544374942779541,
-      "reward_std": 0.5835935473442078,
-      "rewards/rollout_reward_func/mean": 0.544374942779541,
-      "rewards/rollout_reward_func/std": 0.6256605982780457,
-      "sampling/importance_sampling_ratio/max": 1.2402807474136353,
-      "sampling/importance_sampling_ratio/mean": 1.0031979084014893,
-      "sampling/importance_sampling_ratio/min": 0.8138751983642578,
-      "sampling/sampling_logp_difference/max": 0.22644925117492676,
-      "sampling/sampling_logp_difference/mean": 0.00406267587095499,
-      "step": 58,
-      "step_time": 10.877047929000128
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 323.6875,
-      "completions/mean_terminated_length": 323.6875,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.048880314076086506,
-      "epoch": 0.00118,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.33240067958831787,
-      "kl": 2.3523209914565086,
-      "learning_rate": 9.996857996866312e-05,
-      "loss": 0.018,
-      "num_tokens": 3227211.0,
-      "reward": 0.7060937285423279,
-      "reward_std": 0.4677930176258087,
-      "rewards/rollout_reward_func/mean": 0.7060937285423279,
-      "rewards/rollout_reward_func/std": 0.4698605537414551,
-      "sampling/importance_sampling_ratio/max": 1.2279773950576782,
-      "sampling/importance_sampling_ratio/mean": 1.0035088062286377,
-      "sampling/importance_sampling_ratio/min": 0.7834603786468506,
-      "sampling/sampling_logp_difference/max": 0.24589014053344727,
-      "sampling/sampling_logp_difference/mean": 0.0038684559985995293,
-      "step": 59,
-      "step_time": 11.47484337800006
-    },
-    {
-      "clip_ratio/high_max": 0.012500000186264515,
-      "clip_ratio/high_mean": 0.0062500000931322575,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0062500000931322575,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 241.15625,
-      "completions/mean_terminated_length": 241.15625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.08376551068067783,
-      "epoch": 0.0012,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.3866595923900604,
-      "kl": 2.2634962052106857,
-      "learning_rate": 9.996578882141274e-05,
-      "loss": 0.0001,
-      "num_tokens": 3280546.0,
-      "reward": 0.5428124666213989,
-      "reward_std": 0.4759935438632965,
-      "rewards/rollout_reward_func/mean": 0.5428124666213989,
-      "rewards/rollout_reward_func/std": 0.5031015872955322,
-      "sampling/importance_sampling_ratio/max": 1.010900855064392,
-      "sampling/importance_sampling_ratio/mean": 0.984497606754303,
-      "sampling/importance_sampling_ratio/min": 0.7112658023834229,
-      "sampling/sampling_logp_difference/max": 0.2632329761981964,
-      "sampling/sampling_logp_difference/mean": 0.009077915921807289,
-      "step": 60,
-      "step_time": 10.785520703000202
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 365.90625,
-      "completions/mean_terminated_length": 365.90625,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.09243054362013936,
-      "epoch": 0.00122,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.6701250076293945,
-      "kl": 2.195151261985302,
-      "learning_rate": 9.996287897570025e-05,
-      "loss": 0.0417,
-      "num_tokens": 3338666.0,
-      "reward": 0.6145312190055847,
-      "reward_std": 0.5161258578300476,
-      "rewards/rollout_reward_func/mean": 0.6145312190055847,
-      "rewards/rollout_reward_func/std": 0.4940278232097626,
-      "sampling/importance_sampling_ratio/max": 1.4842092990875244,
-      "sampling/importance_sampling_ratio/mean": 1.0207096338272095,
-      "sampling/importance_sampling_ratio/min": 0.7195701599121094,
-      "sampling/sampling_logp_difference/max": 0.3492816388607025,
-      "sampling/sampling_logp_difference/mean": 0.011117234826087952,
-      "step": 61,
-      "step_time": 12.120807182999897
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.013541667023673654,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.013541667023673654,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 713.0,
-      "completions/max_terminated_length": 713.0,
-      "completions/mean_length": 252.6875,
-      "completions/mean_terminated_length": 252.6875,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.18321337562520057,
-      "epoch": 0.00124,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.7521763443946838,
-      "kl": 2.2871460877358913,
-      "learning_rate": 9.995985044074458e-05,
-      "loss": -0.0297,
-      "num_tokens": 3392922.0,
-      "reward": 0.7029687166213989,
-      "reward_std": 0.5108043551445007,
-      "rewards/rollout_reward_func/mean": 0.7029687166213989,
-      "rewards/rollout_reward_func/std": 0.5972934365272522,
-      "sampling/importance_sampling_ratio/max": 1.5505900382995605,
-      "sampling/importance_sampling_ratio/mean": 1.0387890338897705,
-      "sampling/importance_sampling_ratio/min": 0.543106734752655,
-      "sampling/sampling_logp_difference/max": 0.6090604066848755,
-      "sampling/sampling_logp_difference/mean": 0.03258318454027176,
-      "step": 62,
-      "step_time": 10.745307333999563
-    },
-    {
-      "clip_ratio/high_max": 0.0062500000931322575,
-      "clip_ratio/high_mean": 0.0031250000465661287,
-      "clip_ratio/low_mean": 0.004464285913854837,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.007589285960420966,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 686.0,
-      "completions/max_terminated_length": 686.0,
-      "completions/mean_length": 267.1875,
-      "completions/mean_terminated_length": 267.1875,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.2639620187692344,
-      "epoch": 0.00126,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.1851922273635864,
-      "kl": 2.109201803803444,
-      "learning_rate": 9.995670322614067e-05,
-      "loss": 0.0103,
-      "num_tokens": 3448189.0,
-      "reward": 0.38859373331069946,
-      "reward_std": 0.7507745623588562,
-      "rewards/rollout_reward_func/mean": 0.38859373331069946,
-      "rewards/rollout_reward_func/std": 0.8012362122535706,
-      "sampling/importance_sampling_ratio/max": 1.178083062171936,
-      "sampling/importance_sampling_ratio/mean": 0.9361153841018677,
-      "sampling/importance_sampling_ratio/min": 0.36275121569633484,
-      "sampling/sampling_logp_difference/max": 1.0052932500839233,
-      "sampling/sampling_logp_difference/mean": 0.04018761217594147,
-      "step": 63,
-      "step_time": 11.748435048000147
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.013281250139698386,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.013281250139698386,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 704.0,
-      "completions/max_terminated_length": 704.0,
-      "completions/mean_length": 331.1875,
-      "completions/mean_terminated_length": 331.1875,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.28721141861751676,
-      "epoch": 0.00128,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.1359292268753052,
-      "kl": 2.590524345636368,
-      "learning_rate": 9.995343734185947e-05,
-      "loss": -0.0036,
-      "num_tokens": 3503960.0,
-      "reward": 0.19374999403953552,
-      "reward_std": 0.7539432048797607,
-      "rewards/rollout_reward_func/mean": 0.19374999403953552,
-      "rewards/rollout_reward_func/std": 0.7880743741989136,
-      "sampling/importance_sampling_ratio/max": 2.229914426803589,
-      "sampling/importance_sampling_ratio/mean": 1.0449782609939575,
-      "sampling/importance_sampling_ratio/min": 0.6568747162818909,
-      "sampling/sampling_logp_difference/max": 0.8065266609191895,
-      "sampling/sampling_logp_difference/mean": 0.04699143022298813,
-      "step": 64,
-      "step_time": 10.86245570700089
-    },
-    {
-      "clip_ratio/high_max": 0.027724359650164843,
-      "clip_ratio/high_mean": 0.013862179825082421,
-      "clip_ratio/low_mean": 0.012276785913854837,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.02613896573893726,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 713.0,
-      "completions/max_terminated_length": 713.0,
-      "completions/mean_length": 367.84375,
-      "completions/mean_terminated_length": 367.84375,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.2549626398831606,
-      "epoch": 0.0013,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.0763204097747803,
-      "kl": 2.7192279770970345,
-      "learning_rate": 9.995005279824793e-05,
-      "loss": 0.0313,
-      "num_tokens": 3561591.0,
-      "reward": 0.510878324508667,
-      "reward_std": 0.6675535440444946,
-      "rewards/rollout_reward_func/mean": 0.510878324508667,
-      "rewards/rollout_reward_func/std": 0.6865581274032593,
-      "sampling/importance_sampling_ratio/max": 2.2724201679229736,
-      "sampling/importance_sampling_ratio/mean": 1.055070161819458,
-      "sampling/importance_sampling_ratio/min": 0.38922151923179626,
-      "sampling/sampling_logp_difference/max": 0.8026904463768005,
-      "sampling/sampling_logp_difference/mean": 0.04969160258769989,
-      "step": 65,
-      "step_time": 12.01761337899984
-    },
-    {
-      "clip_ratio/high_max": 0.017361111473292112,
-      "clip_ratio/high_mean": 0.008680555736646056,
-      "clip_ratio/low_mean": 0.007589285960420966,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.016269841697067022,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 713.0,
-      "completions/max_terminated_length": 713.0,
-      "completions/mean_length": 328.625,
-      "completions/mean_terminated_length": 328.625,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.28046720940619707,
-      "epoch": 0.00132,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.5655421018600464,
-      "kl": 2.8946786150336266,
-      "learning_rate": 9.994654960602885e-05,
-      "loss": 0.0048,
-      "num_tokens": 3617387.0,
-      "reward": 0.4104095697402954,
-      "reward_std": 0.7785094976425171,
-      "rewards/rollout_reward_func/mean": 0.4104095697402954,
-      "rewards/rollout_reward_func/std": 0.7786465287208557,
-      "sampling/importance_sampling_ratio/max": 2.4785797595977783,
-      "sampling/importance_sampling_ratio/mean": 1.0437313318252563,
-      "sampling/importance_sampling_ratio/min": 0.23077696561813354,
-      "sampling/sampling_logp_difference/max": 1.0955324172973633,
-      "sampling/sampling_logp_difference/mean": 0.03850027546286583,
-      "step": 66,
-      "step_time": 10.966047351000043
-    },
-    {
-      "clip_ratio/high_max": 0.02083333395421505,
-      "clip_ratio/high_mean": 0.010416666977107525,
-      "clip_ratio/low_mean": 0.004464285913854837,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.014880952890962362,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 671.0,
-      "completions/max_terminated_length": 671.0,
-      "completions/mean_length": 291.15625,
-      "completions/mean_terminated_length": 291.15625,
-      "completions/min_length": 6.0,
-      "completions/min_terminated_length": 6.0,
-      "entropy": 0.20491029554978013,
-      "epoch": 0.00134,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.910352349281311,
-      "kl": 3.0426899641752243,
-      "learning_rate": 9.994292777630104e-05,
-      "loss": 0.0459,
-      "num_tokens": 3671217.0,
-      "reward": 0.5428543090820312,
-      "reward_std": 0.645969033241272,
-      "rewards/rollout_reward_func/mean": 0.5428543090820312,
-      "rewards/rollout_reward_func/std": 0.6834079623222351,
-      "sampling/importance_sampling_ratio/max": 1.3474293947219849,
-      "sampling/importance_sampling_ratio/mean": 0.9889612197875977,
-      "sampling/importance_sampling_ratio/min": 0.690324604511261,
-      "sampling/sampling_logp_difference/max": 0.38396239280700684,
-      "sampling/sampling_logp_difference/mean": 0.02743671089410782,
-      "step": 67,
-      "step_time": 11.833512543000097
-    },
-    {
-      "clip_ratio/high_max": 0.010416666977107525,
-      "clip_ratio/high_mean": 0.0052083334885537624,
-      "clip_ratio/low_mean": 0.004464285913854837,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0096726194024086,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 731.0,
-      "completions/max_terminated_length": 731.0,
-      "completions/mean_length": 301.75,
-      "completions/mean_terminated_length": 301.75,
-      "completions/min_length": 4.0,
-      "completions/min_terminated_length": 4.0,
-      "entropy": 0.20781477680429816,
-      "epoch": 0.00136,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.5290205478668213,
-      "kl": 3.803470656275749,
-      "learning_rate": 9.993918732053908e-05,
-      "loss": -0.0065,
-      "num_tokens": 3726569.0,
-      "reward": 0.6554687023162842,
-      "reward_std": 0.5164387226104736,
-      "rewards/rollout_reward_func/mean": 0.6554687023162842,
-      "rewards/rollout_reward_func/std": 0.5465424060821533,
-      "sampling/importance_sampling_ratio/max": 1.9675859212875366,
-      "sampling/importance_sampling_ratio/mean": 0.9724869728088379,
-      "sampling/importance_sampling_ratio/min": 0.5035603642463684,
-      "sampling/sampling_logp_difference/max": 0.6627948880195618,
-      "sampling/sampling_logp_difference/mean": 0.037465643137693405,
-      "step": 68,
-      "step_time": 10.987365917000261
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 704.0,
-      "completions/max_terminated_length": 704.0,
-      "completions/mean_length": 355.34375,
-      "completions/mean_terminated_length": 355.34375,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.20627202972536907,
-      "epoch": 0.00138,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 2.1613802909851074,
-      "kl": 2.834018148481846,
-      "learning_rate": 9.993532825059343e-05,
-      "loss": 0.1375,
-      "num_tokens": 3784772.0,
-      "reward": 0.5009375214576721,
-      "reward_std": 0.6891741752624512,
-      "rewards/rollout_reward_func/mean": 0.5009375214576721,
-      "rewards/rollout_reward_func/std": 0.7470418810844421,
-      "sampling/importance_sampling_ratio/max": 2.173233985900879,
-      "sampling/importance_sampling_ratio/mean": 0.9901520013809204,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 1.214400291442871,
-      "sampling/sampling_logp_difference/mean": 0.034578774124383926,
-      "step": 69,
-      "step_time": 11.697704630000317
-    },
-    {
-      "clip_ratio/high_max": 0.017045455053448677,
-      "clip_ratio/high_mean": 0.008522727526724339,
-      "clip_ratio/low_mean": 0.0062500000931322575,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.014772727619856596,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 686.0,
-      "completions/max_terminated_length": 686.0,
-      "completions/mean_length": 344.4375,
-      "completions/mean_terminated_length": 344.4375,
-      "completions/min_length": 6.0,
-      "completions/min_terminated_length": 6.0,
-      "entropy": 0.20468375319615006,
-      "epoch": 0.0014,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 4.058775424957275,
-      "kl": 5.166271016001701,
-      "learning_rate": 9.993135057869032e-05,
-      "loss": 0.1056,
-      "num_tokens": 3840613.0,
-      "reward": 0.523822009563446,
-      "reward_std": 0.7127509713172913,
-      "rewards/rollout_reward_func/mean": 0.523822009563446,
-      "rewards/rollout_reward_func/std": 0.7505588531494141,
-      "sampling/importance_sampling_ratio/max": 2.7435874938964844,
-      "sampling/importance_sampling_ratio/mean": 0.9111927151679993,
-      "sampling/importance_sampling_ratio/min": 0.055978626012802124,
-      "sampling/sampling_logp_difference/max": 2.868645668029785,
-      "sampling/sampling_logp_difference/mean": 0.07603543251752853,
-      "step": 70,
-      "step_time": 11.120608888999413
-    },
-    {
-      "clip_ratio/high_max": 0.010416666977107525,
-      "clip_ratio/high_mean": 0.0052083334885537624,
-      "clip_ratio/low_mean": 0.00390625,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.009114583488553762,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 309.1875,
-      "completions/mean_terminated_length": 309.1875,
-      "completions/min_length": 6.0,
-      "completions/min_terminated_length": 6.0,
-      "entropy": 0.15981617546640337,
-      "epoch": 0.00142,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.6707774996757507,
-      "kl": 2.870632916688919,
-      "learning_rate": 9.992725431743175e-05,
-      "loss": 0.0433,
-      "num_tokens": 3893310.0,
-      "reward": 0.3804687261581421,
-      "reward_std": 0.775587797164917,
-      "rewards/rollout_reward_func/mean": 0.3804687261581421,
-      "rewards/rollout_reward_func/std": 0.7611953020095825,
-      "sampling/importance_sampling_ratio/max": 1.473890781402588,
-      "sampling/importance_sampling_ratio/mean": 0.970950722694397,
-      "sampling/importance_sampling_ratio/min": 0.20458859205245972,
-      "sampling/sampling_logp_difference/max": 1.586530089378357,
-      "sampling/sampling_logp_difference/mean": 0.023186001926660538,
-      "step": 71,
-      "step_time": 11.624748627000145
-    },
-    {
-      "clip_ratio/high_max": 0.010416666977107525,
-      "clip_ratio/high_mean": 0.0052083334885537624,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0052083334885537624,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 713.0,
-      "completions/max_terminated_length": 713.0,
-      "completions/mean_length": 297.1875,
-      "completions/mean_terminated_length": 297.1875,
-      "completions/min_length": 2.0,
-      "completions/min_terminated_length": 2.0,
-      "entropy": 0.19876525993458927,
-      "epoch": 0.00144,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.8210364580154419,
-      "kl": 3.1162713021039963,
-      "learning_rate": 9.99230394797954e-05,
-      "loss": 0.0235,
-      "num_tokens": 3946157.0,
-      "reward": 0.34984374046325684,
-      "reward_std": 0.8436514735221863,
-      "rewards/rollout_reward_func/mean": 0.34984374046325684,
-      "rewards/rollout_reward_func/std": 0.8514991402626038,
-      "sampling/importance_sampling_ratio/max": 1.324742078781128,
-      "sampling/importance_sampling_ratio/mean": 0.9357403516769409,
-      "sampling/importance_sampling_ratio/min": 0.271320104598999,
-      "sampling/sampling_logp_difference/max": 0.9900331497192383,
-      "sampling/sampling_logp_difference/mean": 0.026520676910877228,
-      "step": 72,
-      "step_time": 11.430621326999471
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 668.0,
-      "completions/max_terminated_length": 668.0,
-      "completions/mean_length": 330.96875,
-      "completions/mean_terminated_length": 330.96875,
-      "completions/min_length": 5.0,
-      "completions/min_terminated_length": 5.0,
-      "entropy": 0.18171639321371913,
-      "epoch": 0.00146,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.9166688919067383,
-      "kl": 3.136391967535019,
-      "learning_rate": 9.991870607913468e-05,
-      "loss": 0.0829,
-      "num_tokens": 4004115.0,
-      "reward": 0.38028740882873535,
-      "reward_std": 0.7326797246932983,
-      "rewards/rollout_reward_func/mean": 0.38028740882873535,
-      "rewards/rollout_reward_func/std": 0.7258565425872803,
-      "sampling/importance_sampling_ratio/max": 1.4462589025497437,
-      "sampling/importance_sampling_ratio/mean": 1.009206771850586,
-      "sampling/importance_sampling_ratio/min": 0.6639372110366821,
-      "sampling/sampling_logp_difference/max": 0.3562343120574951,
-      "sampling/sampling_logp_difference/mean": 0.016143618151545525,
-      "step": 73,
-      "step_time": 10.908971323999822
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 372.75,
-      "completions/mean_terminated_length": 372.75,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "entropy": 0.2190435561351478,
-      "epoch": 0.00148,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.197749376296997,
-      "kl": 2.7420803010463715,
-      "learning_rate": 9.991425412917859e-05,
-      "loss": 0.003,
-      "num_tokens": 4064889.0,
-      "reward": 0.4485345780849457,
-      "reward_std": 0.7104275822639465,
-      "rewards/rollout_reward_func/mean": 0.4485345780849457,
-      "rewards/rollout_reward_func/std": 0.7658042311668396,
-      "sampling/importance_sampling_ratio/max": 1.2594777345657349,
-      "sampling/importance_sampling_ratio/mean": 0.9673911333084106,
-      "sampling/importance_sampling_ratio/min": 0.7373533248901367,
-      "sampling/sampling_logp_difference/max": 0.29490816593170166,
-      "sampling/sampling_logp_difference/mean": 0.017906762659549713,
-      "step": 74,
-      "step_time": 12.366517650999413
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 740.0,
-      "completions/max_terminated_length": 740.0,
-      "completions/mean_length": 293.625,
-      "completions/mean_terminated_length": 293.625,
-      "completions/min_length": 4.0,
-      "completions/min_terminated_length": 4.0,
-      "entropy": 0.1820442157331854,
-      "epoch": 0.0015,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.6407637000083923,
-      "kl": 2.680447168648243,
-      "learning_rate": 9.99096836440317e-05,
-      "loss": 0.0417,
-      "num_tokens": 4118951.0,
-      "reward": 0.3599749207496643,
-      "reward_std": 0.8620452284812927,
-      "rewards/rollout_reward_func/mean": 0.3599749207496643,
-      "rewards/rollout_reward_func/std": 0.8346701264381409,
-      "sampling/importance_sampling_ratio/max": 1.1690596342086792,
-      "sampling/importance_sampling_ratio/mean": 1.002995252609253,
-      "sampling/importance_sampling_ratio/min": 0.8450060486793518,
-      "sampling/sampling_logp_difference/max": 0.2120213508605957,
-      "sampling/sampling_logp_difference/mean": 0.010525684803724289,
-      "step": 75,
-      "step_time": 11.020944403000158
+      "step_time": 18.05020313399973
     }
   ],
   "logging_steps": 1.0,
-  "max_steps": 1800,
-  "num_input_tokens_seen": 4118951,
+  "max_steps": 600,
+  "num_input_tokens_seen": 65998,
   "num_train_epochs": 1,
   "save_steps": 500,
   "stateful_callbacks": {
@@ -2497,7 +55,7 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }