Upload task output 6

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1320014f88906a3527001997c8540a446ca9402530ae0edf38b353c103c38a9b
+oid sha256:3ba99134af9c26de5a00d09c7f018e80a6db7df890f772f6501c19a1e9b14902
 size 4957560304

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f90602f5533e12af45cd2e5c38192a82c4c7c8ac2c8a2de6a820732c6b370840
+oid sha256:719d7f98baf4e7de1424c9ceee137a9b06c378922bb9126b3b6f4d40c380d895
 size 1214366696

trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.022222222222222223,
+  "epoch": 0.04736842105263158,
   "eval_steps": 500,
-  "global_step": 11,
+  "global_step": 45,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -16,31 +16,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 51.0,
-      "completions/max_terminated_length": 51.0,
-      "completions/mean_length": 33.75,
-      "completions/mean_terminated_length": 33.75,
-      "completions/min_length": 7.0,
-      "completions/min_terminated_length": 7.0,
-      "entropy": 0.3139666020870209,
-      "epoch": 0.00202020202020202,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 35.0,
-      "kl": 0.0009818350663408637,
+      "completions/max_length": 39.0,
+      "completions/max_terminated_length": 39.0,
+      "completions/mean_length": 38.0,
+      "completions/mean_terminated_length": 38.0,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 0.4614375829696655,
+      "epoch": 0.0010526315789473684,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 22.875,
+      "kl": 0.002436438575387001,
       "learning_rate": 0.0,
-      "loss": 0.0508,
-      "num_tokens": 7419.0,
-      "reward": -0.06750000268220901,
-      "reward_std": 0.010606599971652031,
-      "rewards/alfworld_rollout_reward_func/mean": -0.06750000268220901,
-      "rewards/alfworld_rollout_reward_func/std": 0.044999998062849045,
-      "sampling/importance_sampling_ratio/max": 1.9397767782211304,
-      "sampling/importance_sampling_ratio/mean": 1.265228271484375,
-      "sampling/importance_sampling_ratio/min": 0.7526623606681824,
-      "sampling/sampling_logp_difference/max": 0.6095893383026123,
-      "sampling/sampling_logp_difference/mean": 0.02095659449696541,
+      "loss": 0.1571,
+      "num_tokens": 8138.0,
+      "reward": -0.10500000417232513,
+      "reward_std": 0.021213199943304062,
+      "rewards/alfworld_rollout_reward_func/mean": -0.10500000417232513,
+      "rewards/alfworld_rollout_reward_func/std": 0.021213199943304062,
+      "sampling/importance_sampling_ratio/max": 0.9498974680900574,
+      "sampling/importance_sampling_ratio/mean": 0.7463880777359009,
+      "sampling/importance_sampling_ratio/min": 0.5428786873817444,
+      "sampling/sampling_logp_difference/max": 0.2791634798049927,
+      "sampling/sampling_logp_difference/mean": 0.02213391289114952,
       "step": 1,
-      "step_time": 33.69074837599874
+      "step_time": 20.640884385999925
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -49,31 +49,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 28.0,
-      "completions/max_terminated_length": 28.0,
-      "completions/mean_length": 19.5,
-      "completions/mean_terminated_length": 19.5,
-      "completions/min_length": 10.0,
-      "completions/min_terminated_length": 10.0,
-      "entropy": 0.13508953154087067,
-      "epoch": 0.00404040404040404,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 12.125,
-      "kl": 0.00010168399603571743,
-      "learning_rate": 1.0000000000000002e-06,
-      "loss": -0.2584,
-      "num_tokens": 14873.0,
-      "reward": 0.4475000202655792,
-      "reward_std": 0.0530330091714859,
-      "rewards/alfworld_rollout_reward_func/mean": 0.4475000202655792,
-      "rewards/alfworld_rollout_reward_func/std": 0.6049448847770691,
-      "sampling/importance_sampling_ratio/max": 1.0083168745040894,
-      "sampling/importance_sampling_ratio/mean": 0.9563820362091064,
-      "sampling/importance_sampling_ratio/min": 0.8328136801719666,
-      "sampling/sampling_logp_difference/max": 0.20131945610046387,
-      "sampling/sampling_logp_difference/mean": 0.008159896358847618,
+      "completions/max_length": 54.0,
+      "completions/max_terminated_length": 54.0,
+      "completions/mean_length": 49.5,
+      "completions/mean_terminated_length": 49.5,
+      "completions/min_length": 45.0,
+      "completions/min_terminated_length": 45.0,
+      "entropy": 0.6891850829124451,
+      "epoch": 0.002105263157894737,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 13.4375,
+      "kl": 0.0023963379207998514,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": -0.0442,
+      "num_tokens": 16245.0,
+      "reward": -0.08500000089406967,
+      "reward_std": 0.007071071770042181,
+      "rewards/alfworld_rollout_reward_func/mean": -0.08500000089406967,
+      "rewards/alfworld_rollout_reward_func/std": 0.007071071770042181,
+      "sampling/importance_sampling_ratio/max": 0.6407822966575623,
+      "sampling/importance_sampling_ratio/mean": 0.5291908383369446,
+      "sampling/importance_sampling_ratio/min": 0.4175994098186493,
+      "sampling/sampling_logp_difference/max": 0.26871776580810547,
+      "sampling/sampling_logp_difference/mean": 0.0313858687877655,
       "step": 2,
-      "step_time": 29.937857670000085
+      "step_time": 19.196953739000037
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -82,31 +82,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 45.0,
-      "completions/max_terminated_length": 45.0,
-      "completions/mean_length": 29.75,
-      "completions/mean_terminated_length": 29.75,
-      "completions/min_length": 9.0,
-      "completions/min_terminated_length": 9.0,
-      "entropy": 0.39794987440109253,
-      "epoch": 0.006060606060606061,
+      "completions/max_length": 32.0,
+      "completions/max_terminated_length": 32.0,
+      "completions/mean_length": 20.0,
+      "completions/mean_terminated_length": 20.0,
+      "completions/min_length": 8.0,
+      "completions/min_terminated_length": 8.0,
+      "entropy": 0.37498730421066284,
+      "epoch": 0.003157894736842105,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 21.875,
-      "kl": 0.0032683000899851322,
-      "learning_rate": 2.0000000000000003e-06,
-      "loss": 0.3206,
-      "num_tokens": 22312.0,
-      "reward": 0.6924999952316284,
-      "reward_std": 0.3429467976093292,
-      "rewards/alfworld_rollout_reward_func/mean": 0.6924999952316284,
-      "rewards/alfworld_rollout_reward_func/std": 0.4912145137786865,
-      "sampling/importance_sampling_ratio/max": 1.1291301250457764,
-      "sampling/importance_sampling_ratio/mean": 0.682624876499176,
-      "sampling/importance_sampling_ratio/min": 0.25211283564567566,
-      "sampling/sampling_logp_difference/max": 1.3662090301513672,
-      "sampling/sampling_logp_difference/mean": 0.03834383189678192,
+      "grad_norm": 22.125,
+      "kl": 0.01113863568753004,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": -0.1476,
+      "num_tokens": 24119.0,
+      "reward": 0.4650000035762787,
+      "reward_std": 0.7283200025558472,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4650000035762787,
+      "rewards/alfworld_rollout_reward_func/std": 0.7283200025558472,
+      "sampling/importance_sampling_ratio/max": 1.0040974617004395,
+      "sampling/importance_sampling_ratio/mean": 0.7581218481063843,
+      "sampling/importance_sampling_ratio/min": 0.5121461749076843,
+      "sampling/sampling_logp_difference/max": 0.2380220890045166,
+      "sampling/sampling_logp_difference/mean": 0.0313568077981472,
       "step": 3,
-      "step_time": 23.65037218500038
+      "step_time": 16.451998752999998
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -115,31 +115,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 51.0,
-      "completions/max_terminated_length": 51.0,
-      "completions/mean_length": 34.5,
-      "completions/mean_terminated_length": 34.5,
-      "completions/min_length": 24.0,
-      "completions/min_terminated_length": 24.0,
-      "entropy": 0.3213953673839569,
-      "epoch": 0.00808080808080808,
+      "completions/max_length": 47.0,
+      "completions/max_terminated_length": 47.0,
+      "completions/mean_length": 45.5,
+      "completions/mean_terminated_length": 45.5,
+      "completions/min_length": 44.0,
+      "completions/min_terminated_length": 44.0,
+      "entropy": 0.5117533206939697,
+      "epoch": 0.004210526315789474,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 14.1875,
-      "kl": 0.00176280050072819,
-      "learning_rate": 3e-06,
-      "loss": -0.0016,
-      "num_tokens": 30170.0,
-      "reward": -0.027499999850988388,
-      "reward_std": 0.01060660183429718,
-      "rewards/alfworld_rollout_reward_func/mean": -0.027499999850988388,
-      "rewards/alfworld_rollout_reward_func/std": 0.022173559293150902,
-      "sampling/importance_sampling_ratio/max": 1.3178173303604126,
-      "sampling/importance_sampling_ratio/mean": 1.11869478225708,
-      "sampling/importance_sampling_ratio/min": 0.9329869747161865,
-      "sampling/sampling_logp_difference/max": 0.207963764667511,
-      "sampling/sampling_logp_difference/mean": 0.014714838936924934,
+      "grad_norm": 19.25,
+      "kl": 0.0021374421194195747,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": -0.0673,
+      "num_tokens": 32189.0,
+      "reward": -0.054999999701976776,
+      "reward_std": 0.0353553369641304,
+      "rewards/alfworld_rollout_reward_func/mean": -0.054999999701976776,
+      "rewards/alfworld_rollout_reward_func/std": 0.0353553369641304,
+      "sampling/importance_sampling_ratio/max": 0.7325339913368225,
+      "sampling/importance_sampling_ratio/mean": 0.6587845087051392,
+      "sampling/importance_sampling_ratio/min": 0.5850350260734558,
+      "sampling/sampling_logp_difference/max": 0.31505775451660156,
+      "sampling/sampling_logp_difference/mean": 0.024996312335133553,
       "step": 4,
-      "step_time": 34.11798172900126
+      "step_time": 20.293132261999972
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -148,31 +148,592 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 42.0,
-      "completions/max_terminated_length": 42.0,
-      "completions/mean_length": 36.75,
-      "completions/mean_terminated_length": 36.75,
+      "completions/max_length": 48.0,
+      "completions/max_terminated_length": 48.0,
+      "completions/mean_length": 41.0,
+      "completions/mean_terminated_length": 41.0,
       "completions/min_length": 34.0,
       "completions/min_terminated_length": 34.0,
-      "entropy": 0.289629191160202,
-      "epoch": 0.010101010101010102,
+      "entropy": 0.4825854003429413,
+      "epoch": 0.005263157894736842,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 17.375,
+      "kl": 0.001994439633563161,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 0.0046,
+      "num_tokens": 40088.0,
+      "reward": -0.014999999664723873,
+      "reward_std": 0.02121320366859436,
+      "rewards/alfworld_rollout_reward_func/mean": -0.014999999664723873,
+      "rewards/alfworld_rollout_reward_func/std": 0.02121320366859436,
+      "sampling/importance_sampling_ratio/max": 1.0312117338180542,
+      "sampling/importance_sampling_ratio/mean": 0.8863861560821533,
+      "sampling/importance_sampling_ratio/min": 0.7415606379508972,
+      "sampling/sampling_logp_difference/max": 0.15862822532653809,
+      "sampling/sampling_logp_difference/mean": 0.019666757434606552,
+      "step": 5,
+      "step_time": 13.858864080000103
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 28.0,
+      "completions/max_terminated_length": 28.0,
+      "completions/mean_length": 17.5,
+      "completions/mean_terminated_length": 17.5,
+      "completions/min_length": 7.0,
+      "completions/min_terminated_length": 7.0,
+      "entropy": 0.2232380509376526,
+      "epoch": 0.00631578947368421,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 23.375,
+      "kl": 0.0005238907760940492,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": -0.5401,
+      "num_tokens": 48022.0,
+      "reward": -0.019999999552965164,
+      "reward_std": 0.01414213515818119,
+      "rewards/alfworld_rollout_reward_func/mean": -0.019999999552965164,
+      "rewards/alfworld_rollout_reward_func/std": 0.01414213515818119,
+      "sampling/importance_sampling_ratio/max": 1.2113770246505737,
+      "sampling/importance_sampling_ratio/mean": 1.105446457862854,
+      "sampling/importance_sampling_ratio/min": 0.9995158910751343,
+      "sampling/sampling_logp_difference/max": 0.11751800775527954,
+      "sampling/sampling_logp_difference/mean": 0.009657654911279678,
+      "step": 6,
+      "step_time": 21.048020786999814
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 33.0,
+      "completions/max_terminated_length": 33.0,
+      "completions/mean_length": 23.0,
+      "completions/mean_terminated_length": 23.0,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "entropy": 0.4719875156879425,
+      "epoch": 0.007368421052631579,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.1767578125,
+      "kl": 0.004416688811033964,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 0.0,
+      "num_tokens": 56010.0,
+      "reward": -0.03999999910593033,
+      "reward_std": 0.0,
+      "rewards/alfworld_rollout_reward_func/mean": -0.03999999910593033,
+      "rewards/alfworld_rollout_reward_func/std": 0.0,
+      "sampling/importance_sampling_ratio/max": 1.3988980054855347,
+      "sampling/importance_sampling_ratio/mean": 1.0634777545928955,
+      "sampling/importance_sampling_ratio/min": 0.7280574440956116,
+      "sampling/sampling_logp_difference/max": 0.5730650424957275,
+      "sampling/sampling_logp_difference/mean": 0.03539224714040756,
+      "step": 7,
+      "step_time": 18.713738823000313
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 62.0,
+      "completions/max_terminated_length": 62.0,
+      "completions/mean_length": 33.0,
+      "completions/mean_terminated_length": 33.0,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "entropy": 0.5133676528930664,
+      "epoch": 0.008421052631578947,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 25.125,
+      "kl": 0.0028432065155357122,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 0.4735,
+      "num_tokens": 63919.0,
+      "reward": -0.10500000417232513,
+      "reward_std": 0.1060660183429718,
+      "rewards/alfworld_rollout_reward_func/mean": -0.10500000417232513,
+      "rewards/alfworld_rollout_reward_func/std": 0.1060660183429718,
+      "sampling/importance_sampling_ratio/max": 0.9997373819351196,
+      "sampling/importance_sampling_ratio/mean": 0.8888552188873291,
+      "sampling/importance_sampling_ratio/min": 0.7779730558395386,
+      "sampling/sampling_logp_difference/max": 0.26956474781036377,
+      "sampling/sampling_logp_difference/mean": 0.02536104992032051,
+      "step": 8,
+      "step_time": 13.82160017599972
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 32.0,
+      "completions/max_terminated_length": 32.0,
+      "completions/mean_length": 31.0,
+      "completions/mean_terminated_length": 31.0,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 0.26874056458473206,
+      "epoch": 0.009473684210526316,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 37.5,
+      "kl": 0.0011579240672290325,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.0041,
+      "num_tokens": 70958.0,
+      "reward": 0.4650000035762787,
+      "reward_std": 0.7566042542457581,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4650000035762787,
+      "rewards/alfworld_rollout_reward_func/std": 0.7566042542457581,
+      "sampling/importance_sampling_ratio/max": 0.9994217157363892,
+      "sampling/importance_sampling_ratio/mean": 0.9738029837608337,
+      "sampling/importance_sampling_ratio/min": 0.9481842517852783,
+      "sampling/sampling_logp_difference/max": 0.053451284766197205,
+      "sampling/sampling_logp_difference/mean": 0.007760646287351847,
+      "step": 9,
+      "step_time": 15.999382474000413
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 25.0,
+      "completions/max_terminated_length": 25.0,
+      "completions/mean_length": 16.0,
+      "completions/mean_terminated_length": 16.0,
+      "completions/min_length": 7.0,
+      "completions/min_terminated_length": 7.0,
+      "entropy": 0.04329242929816246,
+      "epoch": 0.010526315789473684,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.062255859375,
+      "kl": 0.0005357116460800171,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 0.0,
+      "num_tokens": 78925.0,
+      "reward": -0.03999999910593033,
+      "reward_std": 0.0,
+      "rewards/alfworld_rollout_reward_func/mean": -0.03999999910593033,
+      "rewards/alfworld_rollout_reward_func/std": 0.0,
+      "sampling/importance_sampling_ratio/max": 1.2068166732788086,
+      "sampling/importance_sampling_ratio/mean": 1.1034138202667236,
+      "sampling/importance_sampling_ratio/min": 1.0000110864639282,
+      "sampling/sampling_logp_difference/max": 0.17702150344848633,
+      "sampling/sampling_logp_difference/mean": 0.0059349932707846165,
+      "step": 10,
+      "step_time": 17.315315648999785
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 56.0,
+      "completions/max_terminated_length": 56.0,
+      "completions/mean_length": 47.0,
+      "completions/mean_terminated_length": 47.0,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 0.4847353398799896,
+      "epoch": 0.011578947368421053,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 15.9375,
+      "kl": 0.0017729199025779963,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": -0.0421,
+      "num_tokens": 87003.0,
+      "reward": -0.14499999582767487,
+      "reward_std": 0.12020815163850784,
+      "rewards/alfworld_rollout_reward_func/mean": -0.14499999582767487,
+      "rewards/alfworld_rollout_reward_func/std": 0.12020815163850784,
+      "sampling/importance_sampling_ratio/max": 0.9490898847579956,
+      "sampling/importance_sampling_ratio/mean": 0.7465442419052124,
+      "sampling/importance_sampling_ratio/min": 0.5439985394477844,
+      "sampling/sampling_logp_difference/max": 0.25916290283203125,
+      "sampling/sampling_logp_difference/mean": 0.027138864621520042,
+      "step": 11,
+      "step_time": 17.206847251
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 37.0,
+      "completions/max_terminated_length": 37.0,
+      "completions/mean_length": 33.5,
+      "completions/mean_terminated_length": 33.5,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 0.37151485681533813,
+      "epoch": 0.01263157894736842,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 16.375,
-      "kl": 0.0038685458712279797,
+      "grad_norm": 28.75,
+      "kl": 0.0016265626763924956,
+      "learning_rate": 2.2e-06,
+      "loss": -0.0939,
+      "num_tokens": 94868.0,
+      "reward": -0.004999999888241291,
+      "reward_std": 0.007071067579090595,
+      "rewards/alfworld_rollout_reward_func/mean": -0.004999999888241291,
+      "rewards/alfworld_rollout_reward_func/std": 0.007071067579090595,
+      "sampling/importance_sampling_ratio/max": 1.0204386711120605,
+      "sampling/importance_sampling_ratio/mean": 0.9890743494033813,
+      "sampling/importance_sampling_ratio/min": 0.9577099680900574,
+      "sampling/sampling_logp_difference/max": 0.5425167083740234,
+      "sampling/sampling_logp_difference/mean": 0.03503374382853508,
+      "step": 12,
+      "step_time": 19.660785203999694
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 27.0,
+      "completions/max_terminated_length": 27.0,
+      "completions/mean_length": 25.0,
+      "completions/mean_terminated_length": 25.0,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 0.015066479332745075,
+      "epoch": 0.01368421052631579,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.66796875,
+      "kl": 1.8239021528643207e-06,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 0.0566,
+      "num_tokens": 102718.0,
+      "reward": -0.004999999888241291,
+      "reward_std": 0.007071067579090595,
+      "rewards/alfworld_rollout_reward_func/mean": -0.004999999888241291,
+      "rewards/alfworld_rollout_reward_func/std": 0.007071067579090595,
+      "sampling/importance_sampling_ratio/max": 1.0043948888778687,
+      "sampling/importance_sampling_ratio/mean": 1.003443717956543,
+      "sampling/importance_sampling_ratio/min": 1.0024924278259277,
+      "sampling/sampling_logp_difference/max": 0.0052343979477882385,
+      "sampling/sampling_logp_difference/mean": 0.0001914545428007841,
+      "step": 13,
+      "step_time": 20.80306568700007
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 12.0,
+      "completions/max_terminated_length": 12.0,
+      "completions/mean_length": 12.0,
+      "completions/mean_terminated_length": 12.0,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "entropy": 0.001696347608231008,
+      "epoch": 0.014736842105263158,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 2.382876118645072e-10,
+      "kl": 0.0,
+      "learning_rate": 2.6e-06,
+      "loss": 0.0,
+      "num_tokens": 108124.0,
+      "reward": 0.9800000190734863,
+      "reward_std": 0.0,
+      "rewards/alfworld_rollout_reward_func/mean": 0.9800000190734863,
+      "rewards/alfworld_rollout_reward_func/std": 0.0,
+      "sampling/importance_sampling_ratio/max": 1.000008463859558,
+      "sampling/importance_sampling_ratio/mean": 0.999001145362854,
+      "sampling/importance_sampling_ratio/min": 0.9979937672615051,
+      "sampling/sampling_logp_difference/max": 0.0020105750299990177,
+      "sampling/sampling_logp_difference/mean": 8.428060391452163e-05,
+      "step": 14,
+      "step_time": 7.481573125000068
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 31.0,
+      "completions/max_terminated_length": 31.0,
+      "completions/mean_length": 22.5,
+      "completions/mean_terminated_length": 22.5,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "entropy": 0.33394843339920044,
+      "epoch": 0.015789473684210527,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 53.75,
+      "kl": 0.0015290472656488419,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 0.0104,
+      "num_tokens": 116009.0,
+      "reward": -0.02500000037252903,
+      "reward_std": 0.0353553406894207,
+      "rewards/alfworld_rollout_reward_func/mean": -0.02500000037252903,
+      "rewards/alfworld_rollout_reward_func/std": 0.0353553406894207,
+      "sampling/importance_sampling_ratio/max": 1.2525393962860107,
+      "sampling/importance_sampling_ratio/mean": 0.8984261751174927,
+      "sampling/importance_sampling_ratio/min": 0.5443129539489746,
+      "sampling/sampling_logp_difference/max": 0.3489260673522949,
+      "sampling/sampling_logp_difference/mean": 0.03384072706103325,
+      "step": 15,
+      "step_time": 20.938858712000183
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 12.0,
+      "completions/max_terminated_length": 12.0,
+      "completions/mean_length": 12.0,
+      "completions/mean_terminated_length": 12.0,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "entropy": 0.046902913600206375,
+      "epoch": 0.016842105263157894,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.6796875,
+      "kl": 0.009063954465091228,
+      "learning_rate": 3e-06,
+      "loss": 0.0001,
+      "num_tokens": 118750.0,
+      "reward": 1.0,
+      "reward_std": 0.0,
+      "rewards/alfworld_rollout_reward_func/mean": 1.0,
+      "rewards/alfworld_rollout_reward_func/std": 0.0,
+      "sampling/importance_sampling_ratio/max": 0.9999728202819824,
+      "sampling/importance_sampling_ratio/mean": 0.7868670225143433,
+      "sampling/importance_sampling_ratio/min": 0.5737611651420593,
+      "sampling/sampling_logp_difference/max": 0.4196038246154785,
+      "sampling/sampling_logp_difference/mean": 0.023148825392127037,
+      "step": 16,
+      "step_time": 3.7324950440001885
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 106.0,
+      "completions/max_terminated_length": 106.0,
+      "completions/mean_length": 73.0,
+      "completions/mean_terminated_length": 73.0,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 0.7304099202156067,
+      "epoch": 0.017894736842105262,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 30.125,
+      "kl": 0.003262097714468837,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": -0.0417,
+      "num_tokens": 126730.0,
+      "reward": -0.07000000029802322,
+      "reward_std": 0.02828427217900753,
+      "rewards/alfworld_rollout_reward_func/mean": -0.07000000029802322,
+      "rewards/alfworld_rollout_reward_func/std": 0.02828427404165268,
+      "sampling/importance_sampling_ratio/max": 1.774876594543457,
+      "sampling/importance_sampling_ratio/mean": 1.181494951248169,
+      "sampling/importance_sampling_ratio/min": 0.5881131887435913,
+      "sampling/sampling_logp_difference/max": 0.3358616828918457,
+      "sampling/sampling_logp_difference/mean": 0.034129440784454346,
+      "step": 17,
+      "step_time": 14.997703623000234
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 43.0,
+      "completions/max_terminated_length": 43.0,
+      "completions/mean_length": 41.5,
+      "completions/mean_terminated_length": 41.5,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 0.6625626683235168,
+      "epoch": 0.018947368421052633,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 43.75,
+      "kl": 0.005403192713856697,
+      "learning_rate": 3.4000000000000005e-06,
+      "loss": 0.0185,
+      "num_tokens": 134311.0,
+      "reward": -0.14499999582767487,
+      "reward_std": 0.0353553406894207,
+      "rewards/alfworld_rollout_reward_func/mean": -0.14499999582767487,
+      "rewards/alfworld_rollout_reward_func/std": 0.0353553406894207,
+      "sampling/importance_sampling_ratio/max": 1.0006484985351562,
+      "sampling/importance_sampling_ratio/mean": 0.991002082824707,
+      "sampling/importance_sampling_ratio/min": 0.9813557267189026,
+      "sampling/sampling_logp_difference/max": 0.3458864688873291,
+      "sampling/sampling_logp_difference/mean": 0.048138294368982315,
+      "step": 18,
+      "step_time": 24.957764462999876
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 34.0,
+      "completions/max_terminated_length": 34.0,
+      "completions/mean_length": 33.5,
+      "completions/mean_terminated_length": 33.5,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 0.41865402460098267,
+      "epoch": 0.02,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 16.25,
+      "kl": 0.0007216485682874918,
+      "learning_rate": 3.6000000000000003e-06,
+      "loss": 0.0904,
+      "num_tokens": 142471.0,
+      "reward": -0.06000000238418579,
+      "reward_std": 0.04242640733718872,
+      "rewards/alfworld_rollout_reward_func/mean": -0.06000000238418579,
+      "rewards/alfworld_rollout_reward_func/std": 0.04242641106247902,
+      "sampling/importance_sampling_ratio/max": 1.1322691440582275,
+      "sampling/importance_sampling_ratio/mean": 1.019400715827942,
+      "sampling/importance_sampling_ratio/min": 0.9065322279930115,
+      "sampling/sampling_logp_difference/max": 0.3183724880218506,
+      "sampling/sampling_logp_difference/mean": 0.02116771601140499,
+      "step": 19,
+      "step_time": 13.728306028000134
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 30.0,
+      "completions/max_terminated_length": 30.0,
+      "completions/mean_length": 21.5,
+      "completions/mean_terminated_length": 21.5,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "entropy": 0.3202163279056549,
+      "epoch": 0.021052631578947368,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 40.0,
+      "kl": 0.003523568855598569,
+      "learning_rate": 3.8000000000000005e-06,
+      "loss": 0.3062,
+      "num_tokens": 150383.0,
+      "reward": 0.9800000190734863,
+      "reward_std": 0.014142122119665146,
+      "rewards/alfworld_rollout_reward_func/mean": 0.9800000190734863,
+      "rewards/alfworld_rollout_reward_func/std": 0.014142122119665146,
+      "sampling/importance_sampling_ratio/max": 0.8473429679870605,
+      "sampling/importance_sampling_ratio/mean": 0.6803116798400879,
+      "sampling/importance_sampling_ratio/min": 0.5132803320884705,
+      "sampling/sampling_logp_difference/max": 0.6736298203468323,
+      "sampling/sampling_logp_difference/mean": 0.02787277288734913,
+      "step": 20,
+      "step_time": 14.534581864999836
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 13.0,
+      "completions/max_terminated_length": 13.0,
+      "completions/mean_length": 13.0,
+      "completions/mean_terminated_length": 13.0,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "entropy": 0.01933361031115055,
+      "epoch": 0.022105263157894735,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.5625,
+      "kl": 1.6258312825812027e-05,
       "learning_rate": 4.000000000000001e-06,
-      "loss": 0.1093,
-      "num_tokens": 38036.0,
+      "loss": -0.0058,
+      "num_tokens": 155556.0,
+      "reward": 0.9800000190734863,
+      "reward_std": 0.028284287080168724,
+      "rewards/alfworld_rollout_reward_func/mean": 0.9800000190734863,
+      "rewards/alfworld_rollout_reward_func/std": 0.028284287080168724,
+      "sampling/importance_sampling_ratio/max": 0.9930688738822937,
+      "sampling/importance_sampling_ratio/mean": 0.9847794771194458,
+      "sampling/importance_sampling_ratio/min": 0.9764900207519531,
+      "sampling/sampling_logp_difference/max": 0.023764699697494507,
+      "sampling/sampling_logp_difference/mean": 0.0011826909612864256,
+      "step": 21,
+      "step_time": 7.954851187999793
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 41.0,
+      "completions/max_terminated_length": 41.0,
+      "completions/mean_length": 33.5,
+      "completions/mean_terminated_length": 33.5,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 0.5647985339164734,
+      "epoch": 0.023157894736842106,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.09033203125,
+      "kl": 0.003543839557096362,
+      "learning_rate": 4.2000000000000004e-06,
+      "loss": 0.0,
+      "num_tokens": 163481.0,
       "reward": -0.029999999329447746,
-      "reward_std": 0.04242640733718872,
+      "reward_std": 0.0,
       "rewards/alfworld_rollout_reward_func/mean": -0.029999999329447746,
-      "rewards/alfworld_rollout_reward_func/std": 0.053541261702775955,
-      "sampling/importance_sampling_ratio/max": 1.2062911987304688,
-      "sampling/importance_sampling_ratio/mean": 0.9274314641952515,
-      "sampling/importance_sampling_ratio/min": 0.666809618473053,
-      "sampling/sampling_logp_difference/max": 0.4368581771850586,
-      "sampling/sampling_logp_difference/mean": 0.018221663311123848,
-      "step": 5,
-      "step_time": 26.032739881000452
+      "rewards/alfworld_rollout_reward_func/std": 0.0,
+      "sampling/importance_sampling_ratio/max": 0.7573752999305725,
+      "sampling/importance_sampling_ratio/mean": 0.6874363422393799,
+      "sampling/importance_sampling_ratio/min": 0.6174973249435425,
+      "sampling/sampling_logp_difference/max": 0.36206483840942383,
+      "sampling/sampling_logp_difference/mean": 0.039726752787828445,
+      "step": 22,
+      "step_time": 20.54890883600001
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -181,31 +742,130 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 51.0,
-      "completions/max_terminated_length": 51.0,
-      "completions/mean_length": 36.5,
-      "completions/mean_terminated_length": 36.5,
-      "completions/min_length": 27.0,
-      "completions/min_terminated_length": 27.0,
-      "entropy": 0.3298708200454712,
-      "epoch": 0.012121212121212121,
+      "completions/max_length": 40.0,
+      "completions/max_terminated_length": 40.0,
+      "completions/mean_length": 27.5,
+      "completions/mean_terminated_length": 27.5,
+      "completions/min_length": 15.0,
+      "completions/min_terminated_length": 15.0,
+      "entropy": 0.2661120295524597,
+      "epoch": 0.024210526315789474,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 12.625,
-      "kl": 0.0015959894517436624,
+      "grad_norm": 11.1875,
+      "kl": 0.0006570122204720974,
+      "learning_rate": 4.4e-06,
+      "loss": 0.2262,
+      "num_tokens": 170495.0,
+      "reward": 0.4699999988079071,
+      "reward_std": 0.7495331764221191,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4699999988079071,
+      "rewards/alfworld_rollout_reward_func/std": 0.7495331764221191,
+      "sampling/importance_sampling_ratio/max": 1.0002330541610718,
+      "sampling/importance_sampling_ratio/mean": 0.9075675010681152,
+      "sampling/importance_sampling_ratio/min": 0.8149019479751587,
+      "sampling/sampling_logp_difference/max": 0.07020741701126099,
+      "sampling/sampling_logp_difference/mean": 0.006520848721265793,
+      "step": 23,
+      "step_time": 11.645237333000296
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 46.0,
+      "completions/max_terminated_length": 46.0,
+      "completions/mean_length": 29.0,
+      "completions/mean_terminated_length": 29.0,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "entropy": 0.36616265773773193,
+      "epoch": 0.02526315789473684,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 7.6875,
+      "kl": 0.0018788041779771447,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 0.0387,
+      "num_tokens": 176799.0,
+      "reward": 0.4350000023841858,
+      "reward_std": 0.7707464098930359,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4350000023841858,
+      "rewards/alfworld_rollout_reward_func/std": 0.7707464098930359,
+      "sampling/importance_sampling_ratio/max": 0.9999977350234985,
+      "sampling/importance_sampling_ratio/mean": 0.6649609804153442,
+      "sampling/importance_sampling_ratio/min": 0.32992416620254517,
+      "sampling/sampling_logp_difference/max": 0.33374500274658203,
+      "sampling/sampling_logp_difference/mean": 0.02806846797466278,
+      "step": 24,
+      "step_time": 10.383803171000181
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 37.0,
+      "completions/max_terminated_length": 37.0,
+      "completions/mean_length": 35.0,
+      "completions/mean_terminated_length": 35.0,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 0.5878033638000488,
+      "epoch": 0.02631578947368421,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 17.0,
+      "kl": 0.004632506985217333,
+      "learning_rate": 4.800000000000001e-06,
+      "loss": -0.0036,
+      "num_tokens": 184625.0,
+      "reward": -0.004999999888241291,
+      "reward_std": 0.007071067579090595,
+      "rewards/alfworld_rollout_reward_func/mean": -0.004999999888241291,
+      "rewards/alfworld_rollout_reward_func/std": 0.007071067579090595,
+      "sampling/importance_sampling_ratio/max": 0.5404838919639587,
+      "sampling/importance_sampling_ratio/mean": 0.5063132047653198,
+      "sampling/importance_sampling_ratio/min": 0.4721425175666809,
+      "sampling/sampling_logp_difference/max": 0.1909576654434204,
+      "sampling/sampling_logp_difference/mean": 0.037486471235752106,
+      "step": 25,
+      "step_time": 15.916402261999792
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 65.0,
+      "completions/max_terminated_length": 65.0,
+      "completions/mean_length": 49.5,
+      "completions/mean_terminated_length": 49.5,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 0.6504972577095032,
+      "epoch": 0.02736842105263158,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 10.625,
+      "kl": 0.0017027771100401878,
       "learning_rate": 5e-06,
-      "loss": 0.2093,
-      "num_tokens": 45091.0,
-      "reward": -0.054999999701976776,
-      "reward_std": 0.04949747771024704,
-      "rewards/alfworld_rollout_reward_func/mean": -0.054999999701976776,
-      "rewards/alfworld_rollout_reward_func/std": 0.04358898848295212,
-      "sampling/importance_sampling_ratio/max": 1.1252191066741943,
-      "sampling/importance_sampling_ratio/mean": 0.9298492670059204,
-      "sampling/importance_sampling_ratio/min": 0.41582605242729187,
-      "sampling/sampling_logp_difference/max": 0.3034701347351074,
-      "sampling/sampling_logp_difference/mean": 0.018183453008532524,
-      "step": 6,
-      "step_time": 41.60337347300083
+      "loss": -0.1837,
+      "num_tokens": 192397.0,
+      "reward": 0.4650000035762787,
+      "reward_std": 0.6858935952186584,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4650000035762787,
+      "rewards/alfworld_rollout_reward_func/std": 0.6858935952186584,
+      "sampling/importance_sampling_ratio/max": 0.8453776240348816,
+      "sampling/importance_sampling_ratio/mean": 0.44594645500183105,
+      "sampling/importance_sampling_ratio/min": 0.04651529714465141,
+      "sampling/sampling_logp_difference/max": 0.783210277557373,
+      "sampling/sampling_logp_difference/mean": 0.046462640166282654,
+      "step": 26,
+      "step_time": 18.7025417509999
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -214,31 +874,130 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 42.0,
-      "completions/max_terminated_length": 42.0,
-      "completions/mean_length": 35.75,
-      "completions/mean_terminated_length": 35.75,
-      "completions/min_length": 27.0,
-      "completions/min_terminated_length": 27.0,
-      "entropy": 0.4321210980415344,
-      "epoch": 0.014141414141414142,
+      "completions/max_length": 38.0,
+      "completions/max_terminated_length": 38.0,
+      "completions/mean_length": 37.5,
+      "completions/mean_terminated_length": 37.5,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 0.3604365885257721,
+      "epoch": 0.028421052631578948,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 10.1875,
-      "kl": 0.001390452147461474,
-      "learning_rate": 6e-06,
-      "loss": -0.0007,
-      "num_tokens": 52432.0,
-      "reward": -0.029999999329447746,
+      "grad_norm": 16.75,
+      "kl": 0.0014921823749318719,
+      "learning_rate": 5.2e-06,
+      "loss": -0.0412,
+      "num_tokens": 200387.0,
+      "reward": 0.48500001430511475,
+      "reward_std": 0.7283200025558472,
+      "rewards/alfworld_rollout_reward_func/mean": 0.48500001430511475,
+      "rewards/alfworld_rollout_reward_func/std": 0.7283200025558472,
+      "sampling/importance_sampling_ratio/max": 0.9038622379302979,
+      "sampling/importance_sampling_ratio/mean": 0.8569427728652954,
+      "sampling/importance_sampling_ratio/min": 0.8100233674049377,
+      "sampling/sampling_logp_difference/max": 0.3119187355041504,
+      "sampling/sampling_logp_difference/mean": 0.018546080216765404,
+      "step": 27,
+      "step_time": 12.738885511999797
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 43.0,
+      "completions/max_terminated_length": 43.0,
+      "completions/mean_length": 28.5,
+      "completions/mean_terminated_length": 28.5,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "entropy": 0.5427743792533875,
+      "epoch": 0.029473684210526315,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 20.875,
+      "kl": 0.00245861429721117,
+      "learning_rate": 5.400000000000001e-06,
+      "loss": 0.2457,
+      "num_tokens": 206148.0,
+      "reward": 0.45000001788139343,
+      "reward_std": 0.7495332360267639,
+      "rewards/alfworld_rollout_reward_func/mean": 0.45000001788139343,
+      "rewards/alfworld_rollout_reward_func/std": 0.7495331764221191,
+      "sampling/importance_sampling_ratio/max": 0.9932032227516174,
+      "sampling/importance_sampling_ratio/mean": 0.8885658383369446,
+      "sampling/importance_sampling_ratio/min": 0.7839284539222717,
+      "sampling/sampling_logp_difference/max": 0.17587971687316895,
+      "sampling/sampling_logp_difference/mean": 0.023123273625969887,
+      "step": 28,
+      "step_time": 10.170499154000026
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 40.0,
+      "completions/max_terminated_length": 40.0,
+      "completions/mean_length": 26.0,
+      "completions/mean_terminated_length": 26.0,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "entropy": 0.25943219661712646,
+      "epoch": 0.030526315789473683,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 22.125,
+      "kl": 0.003534747986122966,
+      "learning_rate": 5.600000000000001e-06,
+      "loss": -0.3005,
+      "num_tokens": 213920.0,
+      "reward": -0.06499999761581421,
+      "reward_std": 0.02121320180594921,
+      "rewards/alfworld_rollout_reward_func/mean": -0.06499999761581421,
+      "rewards/alfworld_rollout_reward_func/std": 0.02121320180594921,
+      "sampling/importance_sampling_ratio/max": 1.023316740989685,
+      "sampling/importance_sampling_ratio/mean": 0.942709743976593,
+      "sampling/importance_sampling_ratio/min": 0.862102746963501,
+      "sampling/sampling_logp_difference/max": 0.15212655067443848,
+      "sampling/sampling_logp_difference/mean": 0.01525637786835432,
+      "step": 29,
+      "step_time": 21.201686755000082
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 26.0,
+      "completions/max_terminated_length": 26.0,
+      "completions/mean_length": 19.0,
+      "completions/mean_terminated_length": 19.0,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "entropy": 0.3542520999908447,
+      "epoch": 0.031578947368421054,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 40.5,
+      "kl": 0.0019339009886607528,
+      "learning_rate": 5.8e-06,
+      "loss": -0.0148,
+      "num_tokens": 221855.0,
+      "reward": -0.044999998062849045,
       "reward_std": 0.02121320366859436,
-      "rewards/alfworld_rollout_reward_func/mean": -0.029999999329447746,
-      "rewards/alfworld_rollout_reward_func/std": 0.021602468565106392,
-      "sampling/importance_sampling_ratio/max": 1.0466150045394897,
-      "sampling/importance_sampling_ratio/mean": 0.8151911497116089,
-      "sampling/importance_sampling_ratio/min": 0.6927466988563538,
-      "sampling/sampling_logp_difference/max": 0.16335582733154297,
-      "sampling/sampling_logp_difference/mean": 0.017741559073328972,
-      "step": 7,
-      "step_time": 32.74947298400002
+      "rewards/alfworld_rollout_reward_func/mean": -0.044999998062849045,
+      "rewards/alfworld_rollout_reward_func/std": 0.02121320366859436,
+      "sampling/importance_sampling_ratio/max": 1.0256065130233765,
+      "sampling/importance_sampling_ratio/mean": 0.7340742349624634,
+      "sampling/importance_sampling_ratio/min": 0.4425419569015503,
+      "sampling/sampling_logp_difference/max": 0.6727456450462341,
+      "sampling/sampling_logp_difference/mean": 0.03979727625846863,
+      "step": 30,
+      "step_time": 20.55009816499978
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -247,31 +1006,196 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 49.0,
-      "completions/max_terminated_length": 49.0,
-      "completions/mean_length": 33.5,
-      "completions/mean_terminated_length": 33.5,
+      "completions/max_length": 37.0,
+      "completions/max_terminated_length": 37.0,
+      "completions/mean_length": 32.5,
+      "completions/mean_terminated_length": 32.5,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 0.3066657483577728,
+      "epoch": 0.03263157894736842,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 18.25,
+      "kl": 0.0021998020820319653,
+      "learning_rate": 6e-06,
+      "loss": -0.1048,
+      "num_tokens": 229801.0,
+      "reward": -0.03500000014901161,
+      "reward_std": 0.007071067579090595,
+      "rewards/alfworld_rollout_reward_func/mean": -0.03500000014901161,
+      "rewards/alfworld_rollout_reward_func/std": 0.007071067579090595,
+      "sampling/importance_sampling_ratio/max": 1.1379889249801636,
+      "sampling/importance_sampling_ratio/mean": 1.1316804885864258,
+      "sampling/importance_sampling_ratio/min": 1.1253719329833984,
+      "sampling/sampling_logp_difference/max": 0.14957481622695923,
+      "sampling/sampling_logp_difference/mean": 0.012969362549483776,
+      "step": 31,
+      "step_time": 18.629926092999995
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 33.0,
+      "completions/max_terminated_length": 33.0,
+      "completions/mean_length": 29.0,
+      "completions/mean_terminated_length": 29.0,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 0.199268639087677,
+      "epoch": 0.03368421052631579,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 20.625,
+      "kl": 0.002764245495200157,
+      "learning_rate": 6.200000000000001e-06,
+      "loss": -0.2126,
+      "num_tokens": 237852.0,
+      "reward": 0.5,
+      "reward_std": 0.7071067690849304,
+      "rewards/alfworld_rollout_reward_func/mean": 0.5,
+      "rewards/alfworld_rollout_reward_func/std": 0.7071067690849304,
+      "sampling/importance_sampling_ratio/max": 1.083717703819275,
+      "sampling/importance_sampling_ratio/mean": 0.9082245826721191,
+      "sampling/importance_sampling_ratio/min": 0.7327314019203186,
+      "sampling/sampling_logp_difference/max": 0.29439616203308105,
+      "sampling/sampling_logp_difference/mean": 0.009704618714749813,
+      "step": 32,
+      "step_time": 21.648855392000314
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 48.0,
+      "completions/max_terminated_length": 48.0,
+      "completions/mean_length": 26.0,
+      "completions/mean_terminated_length": 26.0,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "entropy": 0.5480604767799377,
+      "epoch": 0.034736842105263156,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 28.125,
+      "kl": 0.001682703266851604,
+      "learning_rate": 6.4000000000000006e-06,
+      "loss": -0.4013,
+      "num_tokens": 245900.0,
+      "reward": -0.04500000178813934,
+      "reward_std": 0.007071068976074457,
+      "rewards/alfworld_rollout_reward_func/mean": -0.04500000178813934,
+      "rewards/alfworld_rollout_reward_func/std": 0.007071068976074457,
+      "sampling/importance_sampling_ratio/max": 1.000064730644226,
+      "sampling/importance_sampling_ratio/mean": 0.8534832000732422,
+      "sampling/importance_sampling_ratio/min": 0.7069017291069031,
+      "sampling/sampling_logp_difference/max": 0.17766308784484863,
+      "sampling/sampling_logp_difference/mean": 0.020473649725317955,
+      "step": 33,
+      "step_time": 14.458168422000199
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 43.0,
+      "completions/max_terminated_length": 43.0,
+      "completions/mean_length": 41.0,
+      "completions/mean_terminated_length": 41.0,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 0.4645077884197235,
+      "epoch": 0.035789473684210524,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 29.0,
+      "kl": 0.010407637804746628,
+      "learning_rate": 6.600000000000001e-06,
+      "loss": -0.1263,
+      "num_tokens": 253797.0,
+      "reward": 0.4750000238418579,
+      "reward_std": 0.7141778469085693,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4750000238418579,
+      "rewards/alfworld_rollout_reward_func/std": 0.7141778469085693,
+      "sampling/importance_sampling_ratio/max": 1.1287206411361694,
+      "sampling/importance_sampling_ratio/mean": 0.9985779523849487,
+      "sampling/importance_sampling_ratio/min": 0.8684353232383728,
+      "sampling/sampling_logp_difference/max": 0.15062963962554932,
+      "sampling/sampling_logp_difference/mean": 0.017277846112847328,
+      "step": 34,
+      "step_time": 15.326947537999786
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 48.0,
+      "completions/max_terminated_length": 48.0,
+      "completions/mean_length": 30.5,
+      "completions/mean_terminated_length": 30.5,
       "completions/min_length": 13.0,
       "completions/min_terminated_length": 13.0,
-      "entropy": 0.4318622052669525,
-      "epoch": 0.01616161616161616,
+      "entropy": 0.4968964755535126,
+      "epoch": 0.03684210526315789,
       "frac_reward_zero_std": 0.0,
       "grad_norm": 28.625,
-      "kl": 0.0031005144119262695,
+      "kl": 0.005605627316981554,
+      "learning_rate": 6.800000000000001e-06,
+      "loss": 0.4802,
+      "num_tokens": 260704.0,
+      "reward": 0.375,
+      "reward_std": 0.8838834762573242,
+      "rewards/alfworld_rollout_reward_func/mean": 0.375,
+      "rewards/alfworld_rollout_reward_func/std": 0.8838834762573242,
+      "sampling/importance_sampling_ratio/max": 1.1130870580673218,
+      "sampling/importance_sampling_ratio/mean": 1.018231749534607,
+      "sampling/importance_sampling_ratio/min": 0.9233765006065369,
+      "sampling/sampling_logp_difference/max": 0.13432645797729492,
+      "sampling/sampling_logp_difference/mean": 0.023711007088422775,
+      "step": 35,
+      "step_time": 18.487307321000117
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 35.0,
+      "completions/max_terminated_length": 35.0,
+      "completions/mean_length": 33.0,
+      "completions/mean_terminated_length": 33.0,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 0.1966192126274109,
+      "epoch": 0.037894736842105266,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 16.875,
+      "kl": 0.00045083509758114815,
       "learning_rate": 7e-06,
-      "loss": -0.1956,
-      "num_tokens": 60101.0,
-      "reward": 0.1850000023841858,
-      "reward_std": 0.39597976207733154,
-      "rewards/alfworld_rollout_reward_func/mean": 0.1850000023841858,
-      "rewards/alfworld_rollout_reward_func/std": 0.4990991950035095,
-      "sampling/importance_sampling_ratio/max": 1.5858244895935059,
-      "sampling/importance_sampling_ratio/mean": 1.2175490856170654,
-      "sampling/importance_sampling_ratio/min": 0.7764706611633301,
-      "sampling/sampling_logp_difference/max": 0.5269544124603271,
-      "sampling/sampling_logp_difference/mean": 0.018533799797296524,
-      "step": 8,
-      "step_time": 26.770345292001366
+      "loss": 0.1086,
+      "num_tokens": 268494.0,
+      "reward": -0.054999999701976776,
+      "reward_std": 0.007071066647768021,
+      "rewards/alfworld_rollout_reward_func/mean": -0.054999999701976776,
+      "rewards/alfworld_rollout_reward_func/std": 0.007071066647768021,
+      "sampling/importance_sampling_ratio/max": 1.1769022941589355,
+      "sampling/importance_sampling_ratio/mean": 1.0869765281677246,
+      "sampling/importance_sampling_ratio/min": 0.9970508813858032,
+      "sampling/sampling_logp_difference/max": 0.15711736679077148,
+      "sampling/sampling_logp_difference/mean": 0.008907869458198547,
+      "step": 36,
+      "step_time": 15.888863867000055
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -280,31 +1204,163 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 48.0,
-      "completions/max_terminated_length": 48.0,
-      "completions/mean_length": 35.75,
-      "completions/mean_terminated_length": 35.75,
-      "completions/min_length": 25.0,
-      "completions/min_terminated_length": 25.0,
-      "entropy": 0.27756568789482117,
-      "epoch": 0.01818181818181818,
+      "completions/max_length": 38.0,
+      "completions/max_terminated_length": 38.0,
+      "completions/mean_length": 36.5,
+      "completions/mean_terminated_length": 36.5,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 0.4277231991291046,
+      "epoch": 0.03894736842105263,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 21.375,
+      "kl": 0.01490151509642601,
+      "learning_rate": 7.2000000000000005e-06,
+      "loss": 0.1649,
+      "num_tokens": 276265.0,
+      "reward": -0.004999999888241291,
+      "reward_std": 0.007071067579090595,
+      "rewards/alfworld_rollout_reward_func/mean": -0.004999999888241291,
+      "rewards/alfworld_rollout_reward_func/std": 0.007071067579090595,
+      "sampling/importance_sampling_ratio/max": 0.9954823851585388,
+      "sampling/importance_sampling_ratio/mean": 0.7917053699493408,
+      "sampling/importance_sampling_ratio/min": 0.5879284143447876,
+      "sampling/sampling_logp_difference/max": 0.26447629928588867,
+      "sampling/sampling_logp_difference/mean": 0.01949181593954563,
+      "step": 37,
+      "step_time": 19.005740269999933
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 50.0,
+      "completions/max_terminated_length": 50.0,
+      "completions/mean_length": 30.5,
+      "completions/mean_terminated_length": 30.5,
+      "completions/min_length": 11.0,
+      "completions/min_terminated_length": 11.0,
+      "entropy": 0.6443904042243958,
+      "epoch": 0.04,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 25.0,
+      "kl": 0.008831760846078396,
+      "learning_rate": 7.4e-06,
+      "loss": 0.336,
+      "num_tokens": 284353.0,
+      "reward": 0.4699999988079071,
+      "reward_std": 0.6788225173950195,
+      "rewards/alfworld_rollout_reward_func/mean": 0.4699999988079071,
+      "rewards/alfworld_rollout_reward_func/std": 0.6788225173950195,
+      "sampling/importance_sampling_ratio/max": 0.9188930988311768,
+      "sampling/importance_sampling_ratio/mean": 0.8503769040107727,
+      "sampling/importance_sampling_ratio/min": 0.7818607091903687,
+      "sampling/sampling_logp_difference/max": 0.3467235565185547,
+      "sampling/sampling_logp_difference/mean": 0.029929999262094498,
+      "step": 38,
+      "step_time": 13.158857492000152
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 38.0,
+      "completions/max_terminated_length": 38.0,
+      "completions/mean_length": 26.5,
+      "completions/mean_terminated_length": 26.5,
+      "completions/min_length": 15.0,
+      "completions/min_terminated_length": 15.0,
+      "entropy": 0.1651572287082672,
+      "epoch": 0.04105263157894737,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 12.5625,
-      "kl": 0.004260518122464418,
+      "grad_norm": 11.5625,
+      "kl": 0.001116986502893269,
+      "learning_rate": 7.600000000000001e-06,
+      "loss": 0.2599,
+      "num_tokens": 290896.0,
+      "reward": 0.9850000143051147,
+      "reward_std": 0.02121318317949772,
+      "rewards/alfworld_rollout_reward_func/mean": 0.9850000143051147,
+      "rewards/alfworld_rollout_reward_func/std": 0.02121318317949772,
+      "sampling/importance_sampling_ratio/max": 1.0000063180923462,
+      "sampling/importance_sampling_ratio/mean": 0.9549174904823303,
+      "sampling/importance_sampling_ratio/min": 0.9098286628723145,
+      "sampling/sampling_logp_difference/max": 0.1355164647102356,
+      "sampling/sampling_logp_difference/mean": 0.007564856205135584,
+      "step": 39,
+      "step_time": 10.67765061099999
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 29.0,
+      "completions/max_terminated_length": 29.0,
+      "completions/mean_length": 18.5,
+      "completions/mean_terminated_length": 18.5,
+      "completions/min_length": 8.0,
+      "completions/min_terminated_length": 8.0,
+      "entropy": 0.008951123803853989,
+      "epoch": 0.042105263157894736,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.0078125,
+      "kl": 1.5400552001665346e-05,
+      "learning_rate": 7.800000000000002e-06,
+      "loss": 0.3981,
+      "num_tokens": 298867.0,
+      "reward": -0.019999999552965164,
+      "reward_std": 0.01414213515818119,
+      "rewards/alfworld_rollout_reward_func/mean": -0.019999999552965164,
+      "rewards/alfworld_rollout_reward_func/std": 0.01414213515818119,
+      "sampling/importance_sampling_ratio/max": 0.9998130798339844,
+      "sampling/importance_sampling_ratio/mean": 0.999461829662323,
+      "sampling/importance_sampling_ratio/min": 0.9991105794906616,
+      "sampling/sampling_logp_difference/max": 0.0008255972061306238,
+      "sampling/sampling_logp_difference/mean": 4.654384247260168e-05,
+      "step": 40,
+      "step_time": 14.952102634999846
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 49.0,
+      "completions/max_terminated_length": 49.0,
+      "completions/mean_length": 30.5,
+      "completions/mean_terminated_length": 30.5,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "entropy": 0.3708825707435608,
+      "epoch": 0.0431578947368421,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.042236328125,
+      "kl": 0.0020869383588433266,
       "learning_rate": 8.000000000000001e-06,
-      "loss": 0.2434,
-      "num_tokens": 67417.0,
-      "reward": 0.4449999928474426,
-      "reward_std": 0.763675332069397,
-      "rewards/alfworld_rollout_reward_func/mean": 0.4449999928474426,
-      "rewards/alfworld_rollout_reward_func/std": 0.6235650777816772,
-      "sampling/importance_sampling_ratio/max": 1.0637807846069336,
-      "sampling/importance_sampling_ratio/mean": 0.9480557441711426,
-      "sampling/importance_sampling_ratio/min": 0.7681576013565063,
-      "sampling/sampling_logp_difference/max": 0.40923213958740234,
-      "sampling/sampling_logp_difference/mean": 0.015672903507947922,
-      "step": 9,
-      "step_time": 30.2497927860004
+      "loss": 0.0,
+      "num_tokens": 306923.0,
+      "reward": -0.12999999523162842,
+      "reward_std": 0.0,
+      "rewards/alfworld_rollout_reward_func/mean": -0.12999999523162842,
+      "rewards/alfworld_rollout_reward_func/std": 0.0,
+      "sampling/importance_sampling_ratio/max": 1.1540521383285522,
+      "sampling/importance_sampling_ratio/mean": 1.1200878620147705,
+      "sampling/importance_sampling_ratio/min": 1.0861235857009888,
+      "sampling/sampling_logp_difference/max": 0.2985405921936035,
+      "sampling/sampling_logp_difference/mean": 0.019144592806696892,
+      "step": 41,
+      "step_time": 14.288663088999783
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -313,136 +1369,136 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 44.0,
-      "completions/max_terminated_length": 44.0,
-      "completions/mean_length": 34.25,
-      "completions/mean_terminated_length": 34.25,
+      "completions/max_length": 37.0,
+      "completions/max_terminated_length": 37.0,
+      "completions/mean_length": 31.5,
+      "completions/mean_terminated_length": 31.5,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 0.2996061444282532,
+      "epoch": 0.04421052631578947,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 11.0,
+      "kl": 0.003400312503799796,
+      "learning_rate": 8.2e-06,
+      "loss": -0.0134,
+      "num_tokens": 314807.0,
+      "reward": -0.07000000029802322,
+      "reward_std": 0.02828427217900753,
+      "rewards/alfworld_rollout_reward_func/mean": -0.07000000029802322,
+      "rewards/alfworld_rollout_reward_func/std": 0.02828427404165268,
+      "sampling/importance_sampling_ratio/max": 0.9828594326972961,
+      "sampling/importance_sampling_ratio/mean": 0.8530155420303345,
+      "sampling/importance_sampling_ratio/min": 0.723171591758728,
+      "sampling/sampling_logp_difference/max": 0.14810717105865479,
+      "sampling/sampling_logp_difference/mean": 0.008185315877199173,
+      "step": 42,
+      "step_time": 22.299509426999975
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 47.0,
+      "completions/max_terminated_length": 47.0,
+      "completions/mean_length": 43.0,
+      "completions/mean_terminated_length": 43.0,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 0.5871202349662781,
+      "epoch": 0.045263157894736845,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 21.125,
+      "kl": 0.03479873761534691,
+      "learning_rate": 8.400000000000001e-06,
+      "loss": 0.0345,
+      "num_tokens": 322899.0,
+      "reward": -0.08500000089406967,
+      "reward_std": 0.007071071770042181,
+      "rewards/alfworld_rollout_reward_func/mean": -0.08500000089406967,
+      "rewards/alfworld_rollout_reward_func/std": 0.007071071770042181,
+      "sampling/importance_sampling_ratio/max": 0.6903160810470581,
+      "sampling/importance_sampling_ratio/mean": 0.6763739585876465,
+      "sampling/importance_sampling_ratio/min": 0.6624317765235901,
+      "sampling/sampling_logp_difference/max": 0.3432796001434326,
+      "sampling/sampling_logp_difference/mean": 0.027056951075792313,
+      "step": 43,
+      "step_time": 12.069133553000029
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 29.0,
+      "completions/max_terminated_length": 29.0,
+      "completions/mean_length": 28.0,
+      "completions/mean_terminated_length": 28.0,
       "completions/min_length": 27.0,
       "completions/min_terminated_length": 27.0,
-      "entropy": 0.44270509481430054,
-      "epoch": 0.020202020202020204,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 13.375,
-      "kl": 0.007183426525443792,
-      "learning_rate": 9e-06,
-      "loss": 0.0556,
-      "num_tokens": 75013.0,
-      "reward": -0.04749999940395355,
-      "reward_std": 0.04596194252371788,
-      "rewards/alfworld_rollout_reward_func/mean": -0.04749999940395355,
-      "rewards/alfworld_rollout_reward_func/std": 0.03862210363149643,
-      "sampling/importance_sampling_ratio/max": 1.077842116355896,
-      "sampling/importance_sampling_ratio/mean": 0.8753722906112671,
-      "sampling/importance_sampling_ratio/min": 0.7739592790603638,
-      "sampling/sampling_logp_difference/max": 0.32768964767456055,
-      "sampling/sampling_logp_difference/mean": 0.020173490047454834,
-      "step": 10,
-      "step_time": 37.64899114999935
-    },
-    {
-      "epoch": 0.020202020202020204,
-      "eval_clip_ratio/high_max": 0.0,
-      "eval_clip_ratio/high_mean": 0.0,
-      "eval_clip_ratio/low_mean": 0.0,
-      "eval_clip_ratio/low_min": 0.0,
-      "eval_clip_ratio/region_mean": 0.0,
-      "eval_completions/clipped_ratio": 0.0,
-      "eval_completions/max_length": 40.4,
-      "eval_completions/max_terminated_length": 40.4,
-      "eval_completions/mean_length": 25.9,
-      "eval_completions/mean_terminated_length": 25.9,
-      "eval_completions/min_length": 14.6,
-      "eval_completions/min_terminated_length": 14.6,
-      "eval_entropy": 0.32100764811038973,
-      "eval_frac_reward_zero_std": 0.3,
-      "eval_kl": 0.02566080801188946,
-      "eval_loss": -0.002513153711333871,
-      "eval_num_tokens": 75013.0,
-      "eval_reward": 0.20800000047311187,
-      "eval_reward_std": 0.09758073398843407,
-      "eval_rewards/alfworld_rollout_reward_func/mean": 0.20800000047311187,
-      "eval_rewards/alfworld_rollout_reward_func/std": 0.11996001489460469,
-      "eval_runtime": 144.7099,
-      "eval_samples_per_second": 0.069,
-      "eval_sampling/importance_sampling_ratio/max": 1.365327286720276,
-      "eval_sampling/importance_sampling_ratio/mean": 1.0571398496627809,
-      "eval_sampling/importance_sampling_ratio/min": 0.7952800989151001,
-      "eval_sampling/sampling_logp_difference/max": 0.2557041585445404,
-      "eval_sampling/sampling_logp_difference/mean": 0.01786487102508545,
-      "eval_steps_per_second": 0.021,
-      "step": 10
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 59.0,
-      "completions/max_terminated_length": 59.0,
-      "completions/mean_length": 32.75,
-      "completions/mean_terminated_length": 32.75,
-      "completions/min_length": 12.0,
-      "completions/min_terminated_length": 12.0,
-      "entropy": 0.5878646373748779,
-      "epoch": 0.022222222222222223,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 8.8125,
-      "kl": 0.04139656201004982,
-      "learning_rate": 1e-05,
-      "loss": 0.0004,
-      "num_tokens": 82638.0,
-      "reward": 0.7325000166893005,
-      "reward_std": 0.37830212712287903,
-      "rewards/alfworld_rollout_reward_func/mean": 0.7325000166893005,
-      "rewards/alfworld_rollout_reward_func/std": 0.5349999666213989,
-      "sampling/importance_sampling_ratio/max": 1.11546790599823,
-      "sampling/importance_sampling_ratio/mean": 0.8177005052566528,
-      "sampling/importance_sampling_ratio/min": 0.6592227816581726,
-      "sampling/sampling_logp_difference/max": 0.29733479022979736,
-      "sampling/sampling_logp_difference/mean": 0.026457972824573517,
-      "step": 11,
-      "step_time": 14.770449581999856
-    },
-    {
-      "epoch": 0.022222222222222223,
-      "eval_clip_ratio/high_max": 0.0,
-      "eval_clip_ratio/high_mean": 0.0,
-      "eval_clip_ratio/low_mean": 0.0,
-      "eval_clip_ratio/low_min": 0.0,
-      "eval_clip_ratio/region_mean": 0.0,
-      "eval_completions/clipped_ratio": 0.0,
-      "eval_completions/max_length": 42.4,
-      "eval_completions/max_terminated_length": 42.4,
-      "eval_completions/mean_length": 35.0,
-      "eval_completions/mean_terminated_length": 35.0,
-      "eval_completions/min_length": 25.4,
-      "eval_completions/min_terminated_length": 25.4,
-      "eval_entropy": 0.3629216194152832,
-      "eval_frac_reward_zero_std": 0.2,
-      "eval_kl": 0.035953672230243684,
-      "eval_loss": 0.1647680401802063,
-      "eval_num_tokens": 82638.0,
-      "eval_reward": 0.24750000312924386,
-      "eval_reward_std": 0.036062445025891066,
-      "eval_rewards/alfworld_rollout_reward_func/mean": 0.24750000312924386,
-      "eval_rewards/alfworld_rollout_reward_func/std": 0.16206994708627462,
-      "eval_runtime": 149.9441,
-      "eval_samples_per_second": 0.067,
-      "eval_sampling/importance_sampling_ratio/max": 1.4588795065879823,
-      "eval_sampling/importance_sampling_ratio/mean": 1.0177773118019104,
-      "eval_sampling/importance_sampling_ratio/min": 0.6548068046569824,
-      "eval_sampling/sampling_logp_difference/max": 0.26115807294845583,
-      "eval_sampling/sampling_logp_difference/mean": 0.01973780132830143,
-      "eval_steps_per_second": 0.02,
-      "step": 11
+      "entropy": 0.4670756757259369,
+      "epoch": 0.04631578947368421,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 42.25,
+      "kl": 0.012733863666653633,
+      "learning_rate": 8.6e-06,
+      "loss": 0.1314,
+      "num_tokens": 330905.0,
+      "reward": -0.009999999776482582,
+      "reward_std": 0.01414213515818119,
+      "rewards/alfworld_rollout_reward_func/mean": -0.009999999776482582,
+      "rewards/alfworld_rollout_reward_func/std": 0.01414213515818119,
+      "sampling/importance_sampling_ratio/max": 1.4860219955444336,
+      "sampling/importance_sampling_ratio/mean": 1.3471593856811523,
+      "sampling/importance_sampling_ratio/min": 1.2082966566085815,
+      "sampling/sampling_logp_difference/max": 0.22103285789489746,
+      "sampling/sampling_logp_difference/mean": 0.023863641545176506,
+      "step": 44,
+      "step_time": 17.839258398999846
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 44.0,
+      "completions/max_terminated_length": 44.0,
+      "completions/mean_length": 37.5,
+      "completions/mean_terminated_length": 37.5,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 0.42853403091430664,
+      "epoch": 0.04736842105263158,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 17.5,
+      "kl": 0.018177319318056107,
+      "learning_rate": 8.8e-06,
+      "loss": -0.0227,
+      "num_tokens": 338809.0,
+      "reward": -0.05000000074505806,
+      "reward_std": 0.0707106813788414,
+      "rewards/alfworld_rollout_reward_func/mean": -0.05000000074505806,
+      "rewards/alfworld_rollout_reward_func/std": 0.0707106813788414,
+      "sampling/importance_sampling_ratio/max": 1.1545162200927734,
+      "sampling/importance_sampling_ratio/mean": 0.9563283920288086,
+      "sampling/importance_sampling_ratio/min": 0.7581405639648438,
+      "sampling/sampling_logp_difference/max": 0.15601181983947754,
+      "sampling/sampling_logp_difference/mean": 0.01713634841144085,
+      "step": 45,
+      "step_time": 17.356416965000335
     }
   ],
   "logging_steps": 1,
-  "max_steps": 990,
-  "num_input_tokens_seen": 82638,
-  "num_train_epochs": 2,
+  "max_steps": 4750,
+  "num_input_tokens_seen": 338809,
+  "num_train_epochs": 5,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -457,7 +1513,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 4,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dbaacced9a934966f6add9a97dc4d94ca89cef08cddf204b4e997bfcb787e7c8
+oid sha256:14430176384731c5b857098c61baf8df7d29a6f023673be18bb04f149d7a0f13
 size 7505