{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.24,
  "eval_steps": 500,
  "global_step": 60,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9999.0,
      "completions/max_terminated_length": 9999.0,
      "completions/mean_length": 4554.453125,
      "completions/mean_terminated_length": 4554.453125,
      "completions/min_length": 551.0,
      "completions/min_terminated_length": 551.0,
      "entropy": 0.31497739674523473,
      "epoch": 0.004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.27671217918395996,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": 0.1899,
      "num_tokens": 332413.0,
      "reward": 0.5768750309944153,
      "reward_std": 0.4988396167755127,
      "rewards/alfworld_rollout_reward_func/mean": 0.5768749713897705,
      "rewards/alfworld_rollout_reward_func/std": 0.5170210003852844,
      "sampling/importance_sampling_ratio/max": 2.406592845916748,
      "sampling/importance_sampling_ratio/mean": 0.6213822364807129,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 55.82071304321289,
      "sampling/sampling_logp_difference/mean": 0.023054132238030434,
      "step": 1,
      "step_time": 293.2645074960601
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8320.0,
      "completions/max_terminated_length": 8320.0,
      "completions/mean_length": 5251.21875,
      "completions/mean_terminated_length": 5251.21875,
      "completions/min_length": 1861.0,
      "completions/min_terminated_length": 1861.0,
      "entropy": 0.3733639745041728,
      "epoch": 0.008,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.26362094283103943,
      "kl": 0.0,
      "learning_rate": 1.5e-06,
      "loss": 0.0961,
      "num_tokens": 704651.0,
      "reward": 0.13640625774860382,
      "reward_std": 0.43152421712875366,
      "rewards/alfworld_rollout_reward_func/mean": 0.13640624284744263,
      "rewards/alfworld_rollout_reward_func/std": 0.4294232726097107,
      "sampling/importance_sampling_ratio/max": 2.881873846054077,
      "sampling/importance_sampling_ratio/mean": 0.4364074170589447,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0964527130126953,
      "sampling/sampling_logp_difference/mean": 0.019750293344259262,
      "step": 2,
      "step_time": 281.28423442596977
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10722.0,
      "completions/max_terminated_length": 10722.0,
      "completions/mean_length": 6977.828125,
      "completions/mean_terminated_length": 6977.828125,
      "completions/min_length": 1154.0,
      "completions/min_terminated_length": 1154.0,
      "entropy": 0.3223036811687052,
      "epoch": 0.012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19866329431533813,
      "kl": 0.0017119673575507477,
      "learning_rate": 3e-06,
      "loss": 0.2902,
      "num_tokens": 1196416.0,
      "reward": 0.24703124165534973,
      "reward_std": 0.4671919345855713,
      "rewards/alfworld_rollout_reward_func/mean": 0.24703125655651093,
      "rewards/alfworld_rollout_reward_func/std": 0.4794430732727051,
      "sampling/importance_sampling_ratio/max": 2.9424545764923096,
      "sampling/importance_sampling_ratio/mean": 0.46759214997291565,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.1991941928863525,
      "sampling/sampling_logp_difference/mean": 0.019809413701295853,
      "step": 3,
      "step_time": 336.2479823809699
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9874.0,
      "completions/max_terminated_length": 9874.0,
      "completions/mean_length": 6289.03125,
      "completions/mean_terminated_length": 6289.03125,
      "completions/min_length": 3976.0,
      "completions/min_terminated_length": 3976.0,
      "entropy": 0.33153627486899495,
      "epoch": 0.016,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20725910365581512,
      "kl": 0.0019219094938307535,
      "learning_rate": 2.9980747607565792e-06,
      "loss": -0.0184,
      "num_tokens": 1636162.0,
      "reward": 0.0007812529802322388,
      "reward_std": 0.17323368787765503,
      "rewards/alfworld_rollout_reward_func/mean": 0.0007812529802322388,
      "rewards/alfworld_rollout_reward_func/std": 0.2191564291715622,
      "sampling/importance_sampling_ratio/max": 2.584482192993164,
      "sampling/importance_sampling_ratio/mean": 0.38795873522758484,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.953186511993408,
      "sampling/sampling_logp_difference/mean": 0.020250197499990463,
      "step": 4,
      "step_time": 313.9296533779707
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11285.0,
      "completions/max_terminated_length": 11285.0,
      "completions/mean_length": 7613.421875,
      "completions/mean_terminated_length": 7613.421875,
      "completions/min_length": 2227.0,
      "completions/min_terminated_length": 2227.0,
      "entropy": 0.34567126259207726,
      "epoch": 0.02,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.17643821239471436,
      "kl": 0.0016085030692920554,
      "learning_rate": 2.9923039850878425e-06,
      "loss": 0.0963,
      "num_tokens": 2170909.0,
      "reward": 0.1875,
      "reward_std": 0.4111865162849426,
      "rewards/alfworld_rollout_reward_func/mean": 0.1875,
      "rewards/alfworld_rollout_reward_func/std": 0.445791631937027,
      "sampling/importance_sampling_ratio/max": 2.329904079437256,
      "sampling/importance_sampling_ratio/mean": 0.4071568250656128,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 51.395050048828125,
      "sampling/sampling_logp_difference/mean": 0.02332460880279541,
      "step": 5,
      "step_time": 396.88119790001656
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10068.0,
      "completions/max_terminated_length": 10068.0,
      "completions/mean_length": 7365.96875,
      "completions/mean_terminated_length": 7365.96875,
      "completions/min_length": 2236.0,
      "completions/min_terminated_length": 2236.0,
      "entropy": 0.33807497285306454,
      "epoch": 0.024,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2105790078639984,
      "kl": 0.002014314475673018,
      "learning_rate": 2.982702486492167e-06,
      "loss": 0.1834,
      "num_tokens": 2690011.0,
      "reward": 0.18296873569488525,
      "reward_std": 0.34179380536079407,
      "rewards/alfworld_rollout_reward_func/mean": 0.18296875059604645,
      "rewards/alfworld_rollout_reward_func/std": 0.4501417279243469,
      "sampling/importance_sampling_ratio/max": 2.981616735458374,
      "sampling/importance_sampling_ratio/mean": 0.4046768844127655,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 5.697381019592285,
      "sampling/sampling_logp_difference/mean": 0.020046193152666092,
      "step": 6,
      "step_time": 348.1233048220165
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9846.0,
      "completions/max_terminated_length": 9846.0,
      "completions/mean_length": 6610.140625,
      "completions/mean_terminated_length": 6610.140625,
      "completions/min_length": 808.0,
      "completions/min_terminated_length": 808.0,
      "entropy": 0.3595086098648608,
      "epoch": 0.028,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.218948632478714,
      "kl": 0.0016301772157021333,
      "learning_rate": 2.969294911878742e-06,
      "loss": 0.0717,
      "num_tokens": 3155044.0,
      "reward": 0.171875,
      "reward_std": 0.41918015480041504,
      "rewards/alfworld_rollout_reward_func/mean": 0.171875,
      "rewards/alfworld_rollout_reward_func/std": 0.42613014578819275,
      "sampling/importance_sampling_ratio/max": 2.5824413299560547,
      "sampling/importance_sampling_ratio/mean": 0.3877456784248352,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.626394271850586,
      "sampling/sampling_logp_difference/mean": 0.020398743450641632,
      "step": 7,
      "step_time": 351.79137006499514
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11160.0,
      "completions/max_terminated_length": 11160.0,
      "completions/mean_length": 7036.953125,
      "completions/mean_terminated_length": 7036.953125,
      "completions/min_length": 1917.0,
      "completions/min_terminated_length": 1917.0,
      "entropy": 0.3341470160521567,
      "epoch": 0.032,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2093495875597,
      "kl": 0.0016681601082382258,
      "learning_rate": 2.9521156782993067e-06,
      "loss": -0.0439,
      "num_tokens": 3658497.0,
      "reward": 0.41718751192092896,
      "reward_std": 0.5516971349716187,
      "rewards/alfworld_rollout_reward_func/mean": 0.41718751192092896,
      "rewards/alfworld_rollout_reward_func/std": 0.5447016954421997,
      "sampling/importance_sampling_ratio/max": 2.776736259460449,
      "sampling/importance_sampling_ratio/mean": 0.4658258259296417,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 56.51785659790039,
      "sampling/sampling_logp_difference/mean": 0.022869806736707687,
      "step": 8,
      "step_time": 329.31609062396456
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9616.0,
      "completions/max_terminated_length": 9616.0,
      "completions/mean_length": 8430.71875,
      "completions/mean_terminated_length": 8430.71875,
      "completions/min_length": 4983.0,
      "completions/min_terminated_length": 4983.0,
      "entropy": 0.3272924036718905,
      "epoch": 0.036,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5628678202629089,
      "kl": 0.0017659663353697397,
      "learning_rate": 2.9312088846000733e-06,
      "loss": -0.2308,
      "num_tokens": 4245423.0,
      "reward": 0.032499998807907104,
      "reward_std": 0.24380016326904297,
      "rewards/alfworld_rollout_reward_func/mean": 0.032499998807907104,
      "rewards/alfworld_rollout_reward_func/std": 0.2736351490020752,
      "sampling/importance_sampling_ratio/max": 2.395029067993164,
      "sampling/importance_sampling_ratio/mean": 0.3855500817298889,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.2965643405914307,
      "sampling/sampling_logp_difference/mean": 0.020408470183610916,
      "step": 9,
      "step_time": 363.9418078730232
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6815.0,
      "completions/max_terminated_length": 6815.0,
      "completions/mean_length": 4436.71875,
      "completions/mean_terminated_length": 4436.71875,
      "completions/min_length": 1645.0,
      "completions/min_terminated_length": 1645.0,
      "entropy": 0.3489171755500138,
      "epoch": 0.04,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.28452208638191223,
      "kl": 0.0015779180102981627,
      "learning_rate": 2.906628198220621e-06,
      "loss": 0.1568,
      "num_tokens": 4563965.0,
      "reward": 0.22546875476837158,
      "reward_std": 0.2872272729873657,
      "rewards/alfworld_rollout_reward_func/mean": 0.22546875476837158,
      "rewards/alfworld_rollout_reward_func/std": 0.4715692698955536,
      "sampling/importance_sampling_ratio/max": 2.9534051418304443,
      "sampling/importance_sampling_ratio/mean": 0.4985596835613251,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.560643196105957,
      "sampling/sampling_logp_difference/mean": 0.020065899938344955,
      "step": 10,
      "step_time": 263.016338995003
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9583.0,
      "completions/max_terminated_length": 9583.0,
      "completions/mean_length": 6498.203125,
      "completions/mean_terminated_length": 6498.203125,
      "completions/min_length": 630.0,
      "completions/min_terminated_length": 630.0,
      "entropy": 0.34817238664254546,
      "epoch": 0.044,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.218714639544487,
      "kl": 0.0016383687889174325,
      "learning_rate": 2.878436717430346e-06,
      "loss": 0.187,
      "num_tokens": 5023626.0,
      "reward": 0.18468749523162842,
      "reward_std": 0.4014522433280945,
      "rewards/alfworld_rollout_reward_func/mean": 0.18468749523162842,
      "rewards/alfworld_rollout_reward_func/std": 0.4519691467285156,
      "sampling/importance_sampling_ratio/max": 2.0133447647094727,
      "sampling/importance_sampling_ratio/mean": 0.48143666982650757,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.304382801055908,
      "sampling/sampling_logp_difference/mean": 0.020289719104766846,
      "step": 11,
      "step_time": 358.22142862899636
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10214.0,
      "completions/max_terminated_length": 10214.0,
      "completions/mean_length": 8234.421875,
      "completions/mean_terminated_length": 8234.421875,
      "completions/min_length": 2923.0,
      "completions/min_terminated_length": 2923.0,
      "entropy": 0.2907608295790851,
      "epoch": 0.048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.26984360814094543,
      "kl": 0.0016117684463097248,
      "learning_rate": 2.846706809356113e-06,
      "loss": 0.0146,
      "num_tokens": 5600485.0,
      "reward": 0.11812499165534973,
      "reward_std": 0.35846489667892456,
      "rewards/alfworld_rollout_reward_func/mean": 0.11812499910593033,
      "rewards/alfworld_rollout_reward_func/std": 0.37784212827682495,
      "sampling/importance_sampling_ratio/max": 2.9857699871063232,
      "sampling/importance_sampling_ratio/mean": 0.5836184024810791,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 57.58735656738281,
      "sampling/sampling_logp_difference/mean": 0.01956798881292343,
      "step": 12,
      "step_time": 354.4028672409913
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5514.0,
      "completions/max_terminated_length": 5514.0,
      "completions/mean_length": 2816.578125,
      "completions/mean_terminated_length": 2816.578125,
      "completions/min_length": 899.0,
      "completions/min_terminated_length": 899.0,
      "entropy": 0.37038205191493034,
      "epoch": 0.052,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21182094514369965,
      "kl": 0.0029761453515675385,
      "learning_rate": 2.811519924216873e-06,
      "loss": -0.0192,
      "num_tokens": 5812938.0,
      "reward": 0.7221874594688416,
      "reward_std": 0.4521871507167816,
      "rewards/alfworld_rollout_reward_func/mean": 0.7221875190734863,
      "rewards/alfworld_rollout_reward_func/std": 0.4576561152935028,
      "sampling/importance_sampling_ratio/max": 2.8400075435638428,
      "sampling/importance_sampling_ratio/mean": 0.3720712661743164,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 54.35240936279297,
      "sampling/sampling_logp_difference/mean": 0.04739231616258621,
      "step": 13,
      "step_time": 176.69907861396496
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11718.0,
      "completions/max_terminated_length": 11718.0,
      "completions/mean_length": 8725.5,
      "completions/mean_terminated_length": 8725.5,
      "completions/min_length": 2386.0,
      "completions/min_terminated_length": 2386.0,
      "entropy": 0.34669697750359774,
      "epoch": 0.056,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2509883940219879,
      "kl": 0.002024810473812977,
      "learning_rate": 2.7729663862421267e-06,
      "loss": 0.3618,
      "num_tokens": 6423274.0,
      "reward": 0.1317187398672104,
      "reward_std": 0.27824532985687256,
      "rewards/alfworld_rollout_reward_func/mean": 0.13171875476837158,
      "rewards/alfworld_rollout_reward_func/std": 0.3909270763397217,
      "sampling/importance_sampling_ratio/max": 2.5765702724456787,
      "sampling/importance_sampling_ratio/mean": 0.4724542498588562,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.3789246082305908,
      "sampling/sampling_logp_difference/mean": 0.021418465301394463,
      "step": 14,
      "step_time": 375.7027179340075
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9823.0,
      "completions/max_terminated_length": 9823.0,
      "completions/mean_length": 3761.203125,
      "completions/mean_terminated_length": 3761.203125,
      "completions/min_length": 710.0,
      "completions/min_terminated_length": 710.0,
      "entropy": 0.2707831831648946,
      "epoch": 0.06,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.18436089158058167,
      "kl": 0.0015840599007788114,
      "learning_rate": 2.731145161810915e-06,
      "loss": 0.0515,
      "num_tokens": 6700631.0,
      "reward": 0.6873437166213989,
      "reward_std": 0.3868533968925476,
      "rewards/alfworld_rollout_reward_func/mean": 0.6873437166213989,
      "rewards/alfworld_rollout_reward_func/std": 0.48384782671928406,
      "sampling/importance_sampling_ratio/max": 2.213698625564575,
      "sampling/importance_sampling_ratio/mean": 0.573298454284668,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 56.33928680419922,
      "sampling/sampling_logp_difference/mean": 0.019625678658485413,
      "step": 15,
      "step_time": 284.1389054669853
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6007.0,
      "completions/max_terminated_length": 6007.0,
      "completions/mean_length": 4786.0,
      "completions/mean_terminated_length": 4786.0,
      "completions/min_length": 1835.0,
      "completions/min_terminated_length": 1835.0,
      "entropy": 0.42686293832957745,
      "epoch": 0.064,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5065476298332214,
      "kl": 0.004679446868976811,
      "learning_rate": 2.6861636054065477e-06,
      "loss": -0.0551,
      "num_tokens": 7042679.0,
      "reward": 0.23718750476837158,
      "reward_std": 0.3366313576698303,
      "rewards/alfworld_rollout_reward_func/mean": 0.23718750476837158,
      "rewards/alfworld_rollout_reward_func/std": 0.4672325551509857,
      "sampling/importance_sampling_ratio/max": 1.987215518951416,
      "sampling/importance_sampling_ratio/mean": 0.3825452923774719,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 48.08681869506836,
      "sampling/sampling_logp_difference/mean": 0.02753990702331066,
      "step": 16,
      "step_time": 253.53596857101365
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 13026.0,
      "completions/max_terminated_length": 13026.0,
      "completions/mean_length": 7801.28125,
      "completions/mean_terminated_length": 7801.28125,
      "completions/min_length": 996.0,
      "completions/min_terminated_length": 996.0,
      "entropy": 0.38106839125975966,
      "epoch": 0.068,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13401441276073456,
      "kl": 0.0019643659798020963,
      "learning_rate": 2.6381371840391863e-06,
      "loss": 0.2576,
      "num_tokens": 7587913.0,
      "reward": 0.08687499165534973,
      "reward_std": 0.31602421402931213,
      "rewards/alfworld_rollout_reward_func/mean": 0.08687499165534973,
      "rewards/alfworld_rollout_reward_func/std": 0.37281376123428345,
      "sampling/importance_sampling_ratio/max": 1.947364330291748,
      "sampling/importance_sampling_ratio/mean": 0.346585750579834,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 50.33766174316406,
      "sampling/sampling_logp_difference/mean": 0.024903230369091034,
      "step": 17,
      "step_time": 438.1667110319977
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7932.0,
      "completions/max_terminated_length": 7932.0,
      "completions/mean_length": 5135.984375,
      "completions/mean_terminated_length": 5135.984375,
      "completions/min_length": 2158.0,
      "completions/min_terminated_length": 2158.0,
      "entropy": 0.38822565879672766,
      "epoch": 0.072,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2989322245121002,
      "kl": 0.001747572790918639,
      "learning_rate": 2.58718918084368e-06,
      "loss": -0.1124,
      "num_tokens": 7953544.0,
      "reward": 0.277031272649765,
      "reward_std": 0.3521837890148163,
      "rewards/alfworld_rollout_reward_func/mean": 0.2770312428474426,
      "rewards/alfworld_rollout_reward_func/std": 0.4784753620624542,
      "sampling/importance_sampling_ratio/max": 2.394239902496338,
      "sampling/importance_sampling_ratio/mean": 0.37263223528862,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.304655075073242,
      "sampling/sampling_logp_difference/mean": 0.02233094722032547,
      "step": 18,
      "step_time": 256.8168452990067
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 14106.0,
      "completions/max_terminated_length": 14106.0,
      "completions/mean_length": 10424.546875,
      "completions/mean_terminated_length": 10424.546875,
      "completions/min_length": 2464.0,
      "completions/min_terminated_length": 2464.0,
      "entropy": 0.3585456218570471,
      "epoch": 0.076,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.16573461890220642,
      "kl": 0.0020609867096936796,
      "learning_rate": 2.53345037861353e-06,
      "loss": 0.1829,
      "num_tokens": 8681771.0,
      "reward": 0.17765626311302185,
      "reward_std": 0.4618713855743408,
      "rewards/alfworld_rollout_reward_func/mean": 0.17765626311302185,
      "rewards/alfworld_rollout_reward_func/std": 0.46532371640205383,
      "sampling/importance_sampling_ratio/max": 2.4118423461914062,
      "sampling/importance_sampling_ratio/mean": 0.23810628056526184,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 54.374839782714844,
      "sampling/sampling_logp_difference/mean": 0.026143547147512436,
      "step": 19,
      "step_time": 470.0736003620259
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6909.0,
      "completions/max_terminated_length": 6909.0,
      "completions/mean_length": 3685.25,
      "completions/mean_terminated_length": 3685.25,
      "completions/min_length": 621.0,
      "completions/min_terminated_length": 621.0,
      "entropy": 0.34135088324546814,
      "epoch": 0.08,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1747477948665619,
      "kl": 0.001954720755747985,
      "learning_rate": 2.477058724083334e-06,
      "loss": 0.1728,
      "num_tokens": 8952283.0,
      "reward": 0.4632812738418579,
      "reward_std": 0.3338242769241333,
      "rewards/alfworld_rollout_reward_func/mean": 0.4632812738418579,
      "rewards/alfworld_rollout_reward_func/std": 0.5396590828895569,
      "sampling/importance_sampling_ratio/max": 2.4462101459503174,
      "sampling/importance_sampling_ratio/mean": 0.5562223792076111,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 57.48880386352539,
      "sampling/sampling_logp_difference/mean": 0.025760911405086517,
      "step": 20,
      "step_time": 243.09781286600628
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9257.0,
      "completions/max_terminated_length": 9257.0,
      "completions/mean_length": 5892.0,
      "completions/mean_terminated_length": 5892.0,
      "completions/min_length": 2008.0,
      "completions/min_terminated_length": 2008.0,
      "entropy": 0.317400851752609,
      "epoch": 0.084,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2818284034729004,
      "kl": 0.0034878615733759943,
      "learning_rate": 2.4181589738214946e-06,
      "loss": -0.1166,
      "num_tokens": 9367611.0,
      "reward": 0.13109375536441803,
      "reward_std": 0.4177722930908203,
      "rewards/alfworld_rollout_reward_func/mean": 0.13109375536441803,
      "rewards/alfworld_rollout_reward_func/std": 0.41239187121391296,
      "sampling/importance_sampling_ratio/max": 2.3060288429260254,
      "sampling/importance_sampling_ratio/mean": 0.40336811542510986,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.0395994186401367,
      "sampling/sampling_logp_difference/mean": 0.01938227005302906,
      "step": 21,
      "step_time": 322.3771183890203
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5404.0,
      "completions/max_terminated_length": 5404.0,
      "completions/mean_length": 2908.671875,
      "completions/mean_terminated_length": 2908.671875,
      "completions/min_length": 504.0,
      "completions/min_terminated_length": 504.0,
      "entropy": 0.33527113823220134,
      "epoch": 0.088,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.28914618492126465,
      "kl": 0.0015201076712401118,
      "learning_rate": 2.3569023226421886e-06,
      "loss": 0.4174,
      "num_tokens": 9586310.0,
      "reward": 0.5940625071525574,
      "reward_std": 0.4888940155506134,
      "rewards/alfworld_rollout_reward_func/mean": 0.5940624475479126,
      "rewards/alfworld_rollout_reward_func/std": 0.5188471078872681,
      "sampling/importance_sampling_ratio/max": 2.96110200881958,
      "sampling/importance_sampling_ratio/mean": 0.5440493822097778,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.2356529235839844,
      "sampling/sampling_logp_difference/mean": 0.02087477594614029,
      "step": 22,
      "step_time": 220.2282471570361
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11264.0,
      "completions/max_terminated_length": 11264.0,
      "completions/mean_length": 4416.765625,
      "completions/mean_terminated_length": 4416.765625,
      "completions/min_length": 548.0,
      "completions/min_terminated_length": 548.0,
      "entropy": 0.37216851580888033,
      "epoch": 0.092,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21720609068870544,
      "kl": 0.0015780055618961342,
      "learning_rate": 2.2934460154904436e-06,
      "loss": -0.1388,
      "num_tokens": 9904855.0,
      "reward": 0.44140625,
      "reward_std": 0.5448532104492188,
      "rewards/alfworld_rollout_reward_func/mean": 0.44140625,
      "rewards/alfworld_rollout_reward_func/std": 0.5440409779548645,
      "sampling/importance_sampling_ratio/max": 2.4073739051818848,
      "sampling/importance_sampling_ratio/mean": 0.5275839567184448,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4305299520492554,
      "sampling/sampling_logp_difference/mean": 0.020915336906909943,
      "step": 23,
      "step_time": 276.8828063480614
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8603.0,
      "completions/max_terminated_length": 8603.0,
      "completions/mean_length": 5866.765625,
      "completions/mean_terminated_length": 5866.765625,
      "completions/min_length": 638.0,
      "completions/min_terminated_length": 638.0,
      "entropy": 0.38163668289780617,
      "epoch": 0.096,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1996874064207077,
      "kl": 0.0022600124138989486,
      "learning_rate": 2.227952943796622e-06,
      "loss": 0.0203,
      "num_tokens": 10321992.0,
      "reward": 0.3020312488079071,
      "reward_std": 0.32399436831474304,
      "rewards/alfworld_rollout_reward_func/mean": 0.3020312488079071,
      "rewards/alfworld_rollout_reward_func/std": 0.4859981834888458,
      "sampling/importance_sampling_ratio/max": 2.0370097160339355,
      "sampling/importance_sampling_ratio/mean": 0.322752445936203,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.6053457260131836,
      "sampling/sampling_logp_difference/mean": 0.023195665329694748,
      "step": 24,
      "step_time": 326.98625825899944
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.015625,
      "completions/max_length": 11175.0,
      "completions/max_terminated_length": 11175.0,
      "completions/mean_length": 8415.28125,
      "completions/mean_terminated_length": 8421.2705078125,
      "completions/min_length": 3084.0,
      "completions/min_terminated_length": 3084.0,
      "entropy": 0.3829211392439902,
      "epoch": 0.1,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.15396606922149658,
      "kl": 0.0019763640011660755,
      "learning_rate": 2.160591227336452e-06,
      "loss": 0.0503,
      "num_tokens": 10909370.0,
      "reward": 0.10234375298023224,
      "reward_std": 0.32216501235961914,
      "rewards/alfworld_rollout_reward_func/mean": 0.10234374552965164,
      "rewards/alfworld_rollout_reward_func/std": 0.3721825182437897,
      "sampling/importance_sampling_ratio/max": 2.063018560409546,
      "sampling/importance_sampling_ratio/mean": 0.26939719915390015,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 6.286721229553223,
      "sampling/sampling_logp_difference/mean": 0.021831313148140907,
      "step": 25,
      "step_time": 399.5593527499732
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6961.0,
      "completions/max_terminated_length": 6961.0,
      "completions/mean_length": 4205.625,
      "completions/mean_terminated_length": 4205.625,
      "completions/min_length": 601.0,
      "completions/min_terminated_length": 601.0,
      "entropy": 0.3060068585909903,
      "epoch": 0.104,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.26218128204345703,
      "kl": 0.0015425982546730665,
      "learning_rate": 2.091533782669978e-06,
      "loss": 0.2487,
      "num_tokens": 11216162.0,
      "reward": 0.43156248331069946,
      "reward_std": 0.4275887608528137,
      "rewards/alfworld_rollout_reward_func/mean": 0.43156248331069946,
      "rewards/alfworld_rollout_reward_func/std": 0.5302896499633789,
      "sampling/importance_sampling_ratio/max": 2.158494472503662,
      "sampling/importance_sampling_ratio/mean": 0.5278518795967102,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.369166374206543,
      "sampling/sampling_logp_difference/mean": 0.019138336181640625,
      "step": 26,
      "step_time": 264.540892480014
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6184.0,
      "completions/max_terminated_length": 6184.0,
      "completions/mean_length": 4046.3125,
      "completions/mean_terminated_length": 4046.3125,
      "completions/min_length": 1216.0,
      "completions/min_terminated_length": 1216.0,
      "entropy": 0.35598616674542427,
      "epoch": 0.108,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.568981409072876,
      "kl": 0.001767738020134857,
      "learning_rate": 2.0209578792672304e-06,
      "loss": 0.764,
      "num_tokens": 11507734.0,
      "reward": 0.2854687571525574,
      "reward_std": 0.27528271079063416,
      "rewards/alfworld_rollout_reward_func/mean": 0.2854687571525574,
      "rewards/alfworld_rollout_reward_func/std": 0.4897940456867218,
      "sampling/importance_sampling_ratio/max": 2.9929444789886475,
      "sampling/importance_sampling_ratio/mean": 0.5946164131164551,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 7.642696380615234,
      "sampling/sampling_logp_difference/mean": 0.021483708173036575,
      "step": 27,
      "step_time": 238.4292279400688
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11353.0,
      "completions/max_terminated_length": 11353.0,
      "completions/mean_length": 6085.71875,
      "completions/mean_terminated_length": 6085.71875,
      "completions/min_length": 442.0,
      "completions/min_terminated_length": 442.0,
      "entropy": 0.34554752754047513,
      "epoch": 0.112,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20419467985630035,
      "kl": 0.0022789983468101127,
      "learning_rate": 1.9490446844600373e-06,
      "loss": 0.0168,
      "num_tokens": 11941124.0,
      "reward": 0.3648437559604645,
      "reward_std": 0.3334695100784302,
      "rewards/alfworld_rollout_reward_func/mean": 0.36484378576278687,
      "rewards/alfworld_rollout_reward_func/std": 0.5057353973388672,
      "sampling/importance_sampling_ratio/max": 2.2758615016937256,
      "sampling/importance_sampling_ratio/mean": 0.6538110375404358,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.1021443605422974,
      "sampling/sampling_logp_difference/mean": 0.02113598957657814,
      "step": 28,
      "step_time": 396.3338410100696
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11143.0,
      "completions/max_terminated_length": 11143.0,
      "completions/mean_length": 8093.265625,
      "completions/mean_terminated_length": 8093.265625,
      "completions/min_length": 2100.0,
      "completions/min_terminated_length": 2100.0,
      "entropy": 0.30172505462542176,
      "epoch": 0.116,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.23860569298267365,
      "kl": 0.0015739148620923515,
      "learning_rate": 1.875978798388081e-06,
      "loss": 0.1597,
      "num_tokens": 12510485.0,
      "reward": 0.2574999928474426,
      "reward_std": 0.27106666564941406,
      "rewards/alfworld_rollout_reward_func/mean": 0.2574999928474426,
      "rewards/alfworld_rollout_reward_func/std": 0.4856676161289215,
      "sampling/importance_sampling_ratio/max": 2.100985527038574,
      "sampling/importance_sampling_ratio/mean": 0.4662199318408966,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.5363552570343018,
      "sampling/sampling_logp_difference/mean": 0.0185923483222723,
      "step": 29,
      "step_time": 398.5051951840287
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11407.0,
      "completions/max_terminated_length": 11407.0,
      "completions/mean_length": 7339.609375,
      "completions/mean_terminated_length": 7339.609375,
      "completions/min_length": 3248.0,
      "completions/min_terminated_length": 3248.0,
      "entropy": 0.30220934841781855,
      "epoch": 0.12,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2525518834590912,
      "kl": 0.0015497524218517356,
      "learning_rate": 1.8019477801329903e-06,
      "loss": 0.0827,
      "num_tokens": 13022268.0,
      "reward": 0.15546876192092896,
      "reward_std": 0.3895842730998993,
      "rewards/alfworld_rollout_reward_func/mean": 0.15546876192092896,
      "rewards/alfworld_rollout_reward_func/std": 0.398253858089447,
      "sampling/importance_sampling_ratio/max": 2.3416731357574463,
      "sampling/importance_sampling_ratio/mean": 0.5051580667495728,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9200248718261719,
      "sampling/sampling_logp_difference/mean": 0.01827174611389637,
      "step": 30,
      "step_time": 324.4753994320199
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 12941.0,
      "completions/max_terminated_length": 12941.0,
      "completions/mean_length": 8962.4375,
      "completions/mean_terminated_length": 8962.4375,
      "completions/min_length": 2241.0,
      "completions/min_terminated_length": 2241.0,
      "entropy": 0.31774380756542087,
      "epoch": 0.124,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.22262242436408997,
      "kl": 0.0023503736083512194,
      "learning_rate": 1.7271416662568652e-06,
      "loss": -0.2728,
      "num_tokens": 13653560.0,
      "reward": 0.3187499940395355,
      "reward_std": 0.46891656517982483,
      "rewards/alfworld_rollout_reward_func/mean": 0.3187500238418579,
      "rewards/alfworld_rollout_reward_func/std": 0.49869832396507263,
      "sampling/importance_sampling_ratio/max": 2.161463975906372,
      "sampling/importance_sampling_ratio/mean": 0.36309731006622314,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.3483941555023193,
      "sampling/sampling_logp_difference/mean": 0.02068442292511463,
      "step": 31,
      "step_time": 430.4778445490374
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7700.0,
      "completions/max_terminated_length": 7700.0,
      "completions/mean_length": 4740.34375,
      "completions/mean_terminated_length": 4740.34375,
      "completions/min_length": 751.0,
      "completions/min_terminated_length": 751.0,
      "entropy": 0.3920795237645507,
      "epoch": 0.128,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3321034610271454,
      "kl": 0.0023466993261536118,
      "learning_rate": 1.6517524829811483e-06,
      "loss": 0.2419,
      "num_tokens": 13993934.0,
      "reward": 0.25953125953674316,
      "reward_std": 0.27672427892684937,
      "rewards/alfworld_rollout_reward_func/mean": 0.25953125953674316,
      "rewards/alfworld_rollout_reward_func/std": 0.4774650037288666,
      "sampling/importance_sampling_ratio/max": 2.515002965927124,
      "sampling/importance_sampling_ratio/mean": 0.5241488218307495,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.658240556716919,
      "sampling/sampling_logp_difference/mean": 0.023516817018389702,
      "step": 32,
      "step_time": 244.8374374089908
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9641.0,
      "completions/max_terminated_length": 9641.0,
      "completions/mean_length": 5815.34375,
      "completions/mean_terminated_length": 5815.34375,
      "completions/min_length": 1387.0,
      "completions/min_terminated_length": 1387.0,
      "entropy": 0.3165481761097908,
      "epoch": 0.132,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2114800661802292,
      "kl": 0.0015353642320405925,
      "learning_rate": 1.5759737532580691e-06,
      "loss": 0.3504,
      "num_tokens": 14409316.0,
      "reward": 0.27562499046325684,
      "reward_std": 0.48886263370513916,
      "rewards/alfworld_rollout_reward_func/mean": 0.27562499046325684,
      "rewards/alfworld_rollout_reward_func/std": 0.5106294751167297,
      "sampling/importance_sampling_ratio/max": 1.6374565362930298,
      "sampling/importance_sampling_ratio/mean": 0.5097586512565613,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.133441925048828,
      "sampling/sampling_logp_difference/mean": 0.01870131492614746,
      "step": 33,
      "step_time": 331.69739585091884
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10654.0,
      "completions/max_terminated_length": 10654.0,
      "completions/mean_length": 7535.609375,
      "completions/mean_terminated_length": 7535.609375,
      "completions/min_length": 2351.0,
      "completions/min_terminated_length": 2351.0,
      "entropy": 0.33530362136662006,
      "epoch": 0.136,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.263776034116745,
      "kl": 0.0015736176555947168,
      "learning_rate": 1.5e-06,
      "loss": 0.1335,
      "num_tokens": 14939979.0,
      "reward": 0.18921875953674316,
      "reward_std": 0.44627484679222107,
      "rewards/alfworld_rollout_reward_func/mean": 0.18921875953674316,
      "rewards/alfworld_rollout_reward_func/std": 0.4631647765636444,
      "sampling/importance_sampling_ratio/max": 2.777794122695923,
      "sampling/importance_sampling_ratio/mean": 0.5164031982421875,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 55.2677001953125,
      "sampling/sampling_logp_difference/mean": 0.021179549396038055,
      "step": 34,
      "step_time": 358.0179667470511
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10147.0,
      "completions/max_terminated_length": 10147.0,
      "completions/mean_length": 5750.734375,
      "completions/mean_terminated_length": 5750.734375,
      "completions/min_length": 2291.0,
      "completions/min_terminated_length": 2291.0,
      "entropy": 0.32300999434664845,
      "epoch": 0.14,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19500228762626648,
      "kl": 0.0016171741226571612,
      "learning_rate": 1.4240262467419312e-06,
      "loss": 0.1209,
      "num_tokens": 15353146.0,
      "reward": 0.2917187511920929,
      "reward_std": 0.40162450075149536,
      "rewards/alfworld_rollout_reward_func/mean": 0.2917187809944153,
      "rewards/alfworld_rollout_reward_func/std": 0.48963722586631775,
      "sampling/importance_sampling_ratio/max": 1.8978744745254517,
      "sampling/importance_sampling_ratio/mean": 0.44630223512649536,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 53.83928298950195,
      "sampling/sampling_logp_difference/mean": 0.021219002082943916,
      "step": 35,
      "step_time": 295.46302833205846
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10553.0,
      "completions/max_terminated_length": 10553.0,
      "completions/mean_length": 7239.625,
      "completions/mean_terminated_length": 7239.625,
      "completions/min_length": 2030.0,
      "completions/min_terminated_length": 2030.0,
      "entropy": 0.31878431234508753,
      "epoch": 0.144,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21166330575942993,
      "kl": 0.0018054660904454067,
      "learning_rate": 1.348247517018852e-06,
      "loss": 0.2801,
      "num_tokens": 15866850.0,
      "reward": 0.2901562452316284,
      "reward_std": 0.47076618671417236,
      "rewards/alfworld_rollout_reward_func/mean": 0.2901562452316284,
      "rewards/alfworld_rollout_reward_func/std": 0.5097291469573975,
      "sampling/importance_sampling_ratio/max": 2.524914503097534,
      "sampling/importance_sampling_ratio/mean": 0.5144106149673462,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.984360694885254,
      "sampling/sampling_logp_difference/mean": 0.020332355052232742,
      "step": 36,
      "step_time": 350.01136623701314
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5913.0,
      "completions/max_terminated_length": 5913.0,
      "completions/mean_length": 4780.84375,
      "completions/mean_terminated_length": 4780.84375,
      "completions/min_length": 579.0,
      "completions/min_terminated_length": 579.0,
      "entropy": 0.36987574864178896,
      "epoch": 0.148,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4266482889652252,
      "kl": 0.002036058562225662,
      "learning_rate": 1.2728583337431355e-06,
      "loss": -0.1097,
      "num_tokens": 16207096.0,
      "reward": 0.07484374940395355,
      "reward_std": 0.2557505965232849,
      "rewards/alfworld_rollout_reward_func/mean": 0.07484374940395355,
      "rewards/alfworld_rollout_reward_func/std": 0.34834298491477966,
      "sampling/importance_sampling_ratio/max": 2.214252233505249,
      "sampling/importance_sampling_ratio/mean": 0.5370051860809326,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.3917155265808105,
      "sampling/sampling_logp_difference/mean": 0.02142323926091194,
      "step": 37,
      "step_time": 247.20503475998703
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6967.0,
      "completions/max_terminated_length": 6967.0,
      "completions/mean_length": 4629.875,
      "completions/mean_terminated_length": 4629.875,
      "completions/min_length": 999.0,
      "completions/min_terminated_length": 999.0,
      "entropy": 0.3442773185670376,
      "epoch": 0.152,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.31260499358177185,
      "kl": 0.0025437602398596937,
      "learning_rate": 1.1980522198670096e-06,
      "loss": 0.0576,
      "num_tokens": 16541488.0,
      "reward": 0.44578126072883606,
      "reward_std": 0.5412331819534302,
      "rewards/alfworld_rollout_reward_func/mean": 0.44578126072883606,
      "rewards/alfworld_rollout_reward_func/std": 0.5354713201522827,
      "sampling/importance_sampling_ratio/max": 2.806023597717285,
      "sampling/importance_sampling_ratio/mean": 0.517693281173706,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0272293090820312,
      "sampling/sampling_logp_difference/mean": 0.020449213683605194,
      "step": 38,
      "step_time": 244.387926218973
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 13688.0,
      "completions/max_terminated_length": 13688.0,
      "completions/mean_length": 8324.359375,
      "completions/mean_terminated_length": 8324.359375,
      "completions/min_length": 3154.0,
      "completions/min_terminated_length": 3154.0,
      "entropy": 0.3674225895665586,
      "epoch": 0.156,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.36787140369415283,
      "kl": 0.0025533495027048048,
      "learning_rate": 1.1240212016119191e-06,
      "loss": 0.3241,
      "num_tokens": 17125223.0,
      "reward": 0.09515625238418579,
      "reward_std": 0.3095976710319519,
      "rewards/alfworld_rollout_reward_func/mean": 0.09515625238418579,
      "rewards/alfworld_rollout_reward_func/std": 0.365721732378006,
      "sampling/importance_sampling_ratio/max": 2.9900293350219727,
      "sampling/importance_sampling_ratio/mean": 0.3794456422328949,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 46.29445266723633,
      "sampling/sampling_logp_difference/mean": 0.027921315282583237,
      "step": 39,
      "step_time": 463.61854148590646
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6432.0,
      "completions/max_terminated_length": 6432.0,
      "completions/mean_length": 4293.5625,
      "completions/mean_terminated_length": 4293.5625,
      "completions/min_length": 552.0,
      "completions/min_terminated_length": 552.0,
      "entropy": 0.38137416914105415,
      "epoch": 0.16,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4678648114204407,
      "kl": 0.001712255700113019,
      "learning_rate": 1.050955315539963e-06,
      "loss": -0.2122,
      "num_tokens": 17436395.0,
      "reward": 0.31953126192092896,
      "reward_std": 0.3283727169036865,
      "rewards/alfworld_rollout_reward_func/mean": 0.31953126192092896,
      "rewards/alfworld_rollout_reward_func/std": 0.510062038898468,
      "sampling/importance_sampling_ratio/max": 2.993114948272705,
      "sampling/importance_sampling_ratio/mean": 0.6158585548400879,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 48.348148345947266,
      "sampling/sampling_logp_difference/mean": 0.023211471736431122,
      "step": 40,
      "step_time": 256.8888109070249
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10452.0,
      "completions/max_terminated_length": 10452.0,
      "completions/mean_length": 7975.96875,
      "completions/mean_terminated_length": 7975.96875,
      "completions/min_length": 1643.0,
      "completions/min_terminated_length": 1643.0,
      "entropy": 0.3266838057897985,
      "epoch": 0.164,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2977025508880615,
      "kl": 0.0014619105677411426,
      "learning_rate": 9.790421207327699e-07,
      "loss": 0.0016,
      "num_tokens": 17994953.0,
      "reward": 0.12093749642372131,
      "reward_std": 0.26076871156692505,
      "rewards/alfworld_rollout_reward_func/mean": 0.12093749642372131,
      "rewards/alfworld_rollout_reward_func/std": 0.37234872579574585,
      "sampling/importance_sampling_ratio/max": 2.8854687213897705,
      "sampling/importance_sampling_ratio/mean": 0.5439540147781372,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.1864867210388184,
      "sampling/sampling_logp_difference/mean": 0.01889047771692276,
      "step": 41,
      "step_time": 425.9135665790236
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10013.0,
      "completions/max_terminated_length": 10013.0,
      "completions/mean_length": 5837.21875,
      "completions/mean_terminated_length": 5837.21875,
      "completions/min_length": 812.0,
      "completions/min_terminated_length": 812.0,
      "entropy": 0.3718068078160286,
      "epoch": 0.168,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.27091944217681885,
      "kl": 0.0016485519881825894,
      "learning_rate": 9.084662173300225e-07,
      "loss": -0.1063,
      "num_tokens": 18409943.0,
      "reward": 0.32343751192092896,
      "reward_std": 0.5115345120429993,
      "rewards/alfworld_rollout_reward_func/mean": 0.32343751192092896,
      "rewards/alfworld_rollout_reward_func/std": 0.5089429616928101,
      "sampling/importance_sampling_ratio/max": 2.761387586593628,
      "sampling/importance_sampling_ratio/mean": 0.5661083459854126,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.2433140277862549,
      "sampling/sampling_logp_difference/mean": 0.020694797858595848,
      "step": 42,
      "step_time": 350.4842578980315
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9433.0,
      "completions/max_terminated_length": 9433.0,
      "completions/mean_length": 7310.640625,
      "completions/mean_terminated_length": 7310.640625,
      "completions/min_length": 2144.0,
      "completions/min_terminated_length": 2144.0,
      "entropy": 0.3453982062637806,
      "epoch": 0.172,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3637518882751465,
      "kl": 0.002296481339726597,
      "learning_rate": 8.394087726635485e-07,
      "loss": -0.1358,
      "num_tokens": 18925600.0,
      "reward": 0.2357812523841858,
      "reward_std": 0.44348853826522827,
      "rewards/alfworld_rollout_reward_func/mean": 0.2357812523841858,
      "rewards/alfworld_rollout_reward_func/std": 0.4680924117565155,
      "sampling/importance_sampling_ratio/max": 2.9779891967773438,
      "sampling/importance_sampling_ratio/mean": 0.5728085041046143,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.363124132156372,
      "sampling/sampling_logp_difference/mean": 0.021319666877388954,
      "step": 43,
      "step_time": 331.5770332479151
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8984.0,
      "completions/max_terminated_length": 8984.0,
      "completions/mean_length": 6977.109375,
      "completions/mean_terminated_length": 6977.109375,
      "completions/min_length": 3575.0,
      "completions/min_terminated_length": 3575.0,
      "entropy": 0.36568512208759785,
      "epoch": 0.176,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20232941210269928,
      "kl": 0.0016346415723091923,
      "learning_rate": 7.720470562033787e-07,
      "loss": -0.0301,
      "num_tokens": 19414471.0,
      "reward": 0.04781249910593033,
      "reward_std": 0.2049415409564972,
      "rewards/alfworld_rollout_reward_func/mean": 0.04781249910593033,
      "rewards/alfworld_rollout_reward_func/std": 0.2761898934841156,
      "sampling/importance_sampling_ratio/max": 2.573187828063965,
      "sampling/importance_sampling_ratio/mean": 0.40001991391181946,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.278357982635498,
      "sampling/sampling_logp_difference/mean": 0.02066657692193985,
      "step": 44,
      "step_time": 335.4528454130341
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 15070.0,
      "completions/max_terminated_length": 15070.0,
      "completions/mean_length": 8883.3125,
      "completions/mean_terminated_length": 8883.3125,
      "completions/min_length": 4182.0,
      "completions/min_terminated_length": 4182.0,
      "entropy": 0.31764299469068646,
      "epoch": 0.18,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1879281848669052,
      "kl": 0.0016870554245542735,
      "learning_rate": 7.065539845095568e-07,
      "loss": 0.205,
      "num_tokens": 20034107.0,
      "reward": 0.10062500834465027,
      "reward_std": 0.3260836601257324,
      "rewards/alfworld_rollout_reward_func/mean": 0.10062500089406967,
      "rewards/alfworld_rollout_reward_func/std": 0.3762761652469635,
      "sampling/importance_sampling_ratio/max": 2.5225887298583984,
      "sampling/importance_sampling_ratio/mean": 0.5068827271461487,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.621875524520874,
      "sampling/sampling_logp_difference/mean": 0.019450519233942032,
      "step": 45,
      "step_time": 501.09887950192206
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9683.0,
      "completions/max_terminated_length": 9683.0,
      "completions/mean_length": 6441.609375,
      "completions/mean_terminated_length": 6441.609375,
      "completions/min_length": 2937.0,
      "completions/min_terminated_length": 2937.0,
      "entropy": 0.3502948391251266,
      "epoch": 0.184,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1859540343284607,
      "kl": 0.0018878296577895526,
      "learning_rate": 6.430976773578113e-07,
      "loss": 0.0278,
      "num_tokens": 20485858.0,
      "reward": 0.0010937508195638657,
      "reward_std": 0.23143835365772247,
      "rewards/alfworld_rollout_reward_func/mean": 0.0010937508195638657,
      "rewards/alfworld_rollout_reward_func/std": 0.2593160569667816,
      "sampling/importance_sampling_ratio/max": 2.3244879245758057,
      "sampling/importance_sampling_ratio/mean": 0.34531980752944946,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0072948932647705,
      "sampling/sampling_logp_difference/mean": 0.020865384489297867,
      "step": 46,
      "step_time": 366.95768191007664
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5392.0,
      "completions/max_terminated_length": 5392.0,
      "completions/mean_length": 3591.0625,
      "completions/mean_terminated_length": 3591.0625,
      "completions/min_length": 517.0,
      "completions/min_terminated_length": 517.0,
      "entropy": 0.3532397085800767,
      "epoch": 0.188,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.26849785447120667,
      "kl": 0.0017250235250685364,
      "learning_rate": 5.818410261785057e-07,
      "loss": 0.1737,
      "num_tokens": 20748838.0,
      "reward": 0.41593751311302185,
      "reward_std": 0.5181204080581665,
      "rewards/alfworld_rollout_reward_func/mean": 0.41593748331069946,
      "rewards/alfworld_rollout_reward_func/std": 0.5215962529182434,
      "sampling/importance_sampling_ratio/max": 2.7028884887695312,
      "sampling/importance_sampling_ratio/mean": 0.5614103078842163,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.718446731567383,
      "sampling/sampling_logp_difference/mean": 0.021135177463293076,
      "step": 47,
      "step_time": 194.4773051920347
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5232.0,
      "completions/max_terminated_length": 5232.0,
      "completions/mean_length": 2274.453125,
      "completions/mean_terminated_length": 2274.453125,
      "completions/min_length": 498.0,
      "completions/min_terminated_length": 498.0,
      "entropy": 0.3223302629776299,
      "epoch": 0.192,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20919622480869293,
      "kl": 0.001539721906738123,
      "learning_rate": 5.22941275916667e-07,
      "loss": 0.3358,
      "num_tokens": 20925091.0,
      "reward": 0.6996874809265137,
      "reward_std": 0.4721481502056122,
      "rewards/alfworld_rollout_reward_func/mean": 0.6996875405311584,
      "rewards/alfworld_rollout_reward_func/std": 0.4819766581058502,
      "sampling/importance_sampling_ratio/max": 2.5296757221221924,
      "sampling/importance_sampling_ratio/mean": 0.6417877674102783,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.344658613204956,
      "sampling/sampling_logp_difference/mean": 0.019674479961395264,
      "step": 48,
      "step_time": 168.07057143107522
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6957.0,
      "completions/max_terminated_length": 6957.0,
      "completions/mean_length": 5750.375,
      "completions/mean_terminated_length": 5750.375,
      "completions/min_length": 3849.0,
      "completions/min_terminated_length": 3849.0,
      "entropy": 0.31901704519987106,
      "epoch": 0.196,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.28379565477371216,
      "kl": 0.0021226651278993813,
      "learning_rate": 4.6654962138647007e-07,
      "loss": 0.0489,
      "num_tokens": 21330971.0,
      "reward": 0.02437499538064003,
      "reward_std": 0.186305969953537,
      "rewards/alfworld_rollout_reward_func/mean": 0.02437499724328518,
      "rewards/alfworld_rollout_reward_func/std": 0.2512942850589752,
      "sampling/importance_sampling_ratio/max": 2.9523849487304688,
      "sampling/importance_sampling_ratio/mean": 0.4695657193660736,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 53.46038055419922,
      "sampling/sampling_logp_difference/mean": 0.019282279536128044,
      "step": 49,
      "step_time": 266.79126943598385
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9327.0,
      "completions/max_terminated_length": 9327.0,
      "completions/mean_length": 5568.34375,
      "completions/mean_terminated_length": 5568.34375,
      "completions/min_length": 544.0,
      "completions/min_terminated_length": 544.0,
      "entropy": 0.3215307700447738,
      "epoch": 0.2,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.319042831659317,
      "kl": 0.0018373212151345797,
      "learning_rate": 4.1281081915632036e-07,
      "loss": 0.3955,
      "num_tokens": 21725489.0,
      "reward": 0.22624999284744263,
      "reward_std": 0.40438205003738403,
      "rewards/alfworld_rollout_reward_func/mean": 0.22624999284744263,
      "rewards/alfworld_rollout_reward_func/std": 0.4729659855365753,
      "sampling/importance_sampling_ratio/max": 2.5651261806488037,
      "sampling/importance_sampling_ratio/mean": 0.5718743801116943,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8857594728469849,
      "sampling/sampling_logp_difference/mean": 0.019382823258638382,
      "step": 50,
      "step_time": 338.1545116149937
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11776.0,
      "completions/max_terminated_length": 11776.0,
      "completions/mean_length": 8995.296875,
      "completions/mean_terminated_length": 8995.296875,
      "completions/min_length": 2626.0,
      "completions/min_terminated_length": 2626.0,
      "entropy": 0.35890309792011976,
      "epoch": 0.204,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.16795441508293152,
      "kl": 0.002312422751856502,
      "learning_rate": 3.618628159608137e-07,
      "loss": 0.1271,
      "num_tokens": 22352804.0,
      "reward": 0.06656250357627869,
      "reward_std": 0.3205060660839081,
      "rewards/alfworld_rollout_reward_func/mean": 0.06656249612569809,
      "rewards/alfworld_rollout_reward_func/std": 0.32436618208885193,
      "sampling/importance_sampling_ratio/max": 2.878086566925049,
      "sampling/importance_sampling_ratio/mean": 0.4224788546562195,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.016578197479248,
      "sampling/sampling_logp_difference/mean": 0.022725660353899002,
      "step": 51,
      "step_time": 381.1358364249754
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6659.0,
      "completions/max_terminated_length": 6659.0,
      "completions/mean_length": 4975.1875,
      "completions/mean_terminated_length": 4975.1875,
      "completions/min_length": 2482.0,
      "completions/min_terminated_length": 2482.0,
      "entropy": 0.3571747145615518,
      "epoch": 0.208,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19965247809886932,
      "kl": 0.001949786972545553,
      "learning_rate": 3.1383639459345236e-07,
      "loss": -0.0391,
      "num_tokens": 22705936.0,
      "reward": 0.10843749344348907,
      "reward_std": 0.2537260055541992,
      "rewards/alfworld_rollout_reward_func/mean": 0.10843749344348907,
      "rewards/alfworld_rollout_reward_func/std": 0.3537294268608093,
      "sampling/importance_sampling_ratio/max": 2.9369618892669678,
      "sampling/importance_sampling_ratio/mean": 0.33827975392341614,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.243539333343506,
      "sampling/sampling_logp_difference/mean": 0.020905133336782455,
      "step": 52,
      "step_time": 256.6296185880783
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9607.0,
      "completions/max_terminated_length": 9607.0,
      "completions/mean_length": 5973.78125,
      "completions/mean_terminated_length": 5973.78125,
      "completions/min_length": 1849.0,
      "completions/min_terminated_length": 1849.0,
      "entropy": 0.3052480248734355,
      "epoch": 0.212,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.27389758825302124,
      "kl": 0.0034342061553616077,
      "learning_rate": 2.688548381890859e-07,
      "loss": 0.0311,
      "num_tokens": 23130466.0,
      "reward": 0.19437500834465027,
      "reward_std": 0.4533996284008026,
      "rewards/alfworld_rollout_reward_func/mean": 0.19437500834465027,
      "rewards/alfworld_rollout_reward_func/std": 0.45724809169769287,
      "sampling/importance_sampling_ratio/max": 2.6141345500946045,
      "sampling/importance_sampling_ratio/mean": 0.41237422823905945,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.5475544929504395,
      "sampling/sampling_logp_difference/mean": 0.020619163289666176,
      "step": 53,
      "step_time": 327.58865647405037
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9492.0,
      "completions/max_terminated_length": 9492.0,
      "completions/mean_length": 6306.28125,
      "completions/mean_terminated_length": 6306.28125,
      "completions/min_length": 1341.0,
      "completions/min_terminated_length": 1341.0,
      "entropy": 0.3394437450915575,
      "epoch": 0.216,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19828611612319946,
      "kl": 0.0016102603804029059,
      "learning_rate": 2.2703361375787346e-07,
      "loss": 0.3936,
      "num_tokens": 23577428.0,
      "reward": 0.16374999284744263,
      "reward_std": 0.4402611553668976,
      "rewards/alfworld_rollout_reward_func/mean": 0.16374999284744263,
      "rewards/alfworld_rollout_reward_func/std": 0.44003063440322876,
      "sampling/importance_sampling_ratio/max": 2.7105014324188232,
      "sampling/importance_sampling_ratio/mean": 0.5148346424102783,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.6567630767822266,
      "sampling/sampling_logp_difference/mean": 0.019922319799661636,
      "step": 54,
      "step_time": 330.50816768797813
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5940.0,
      "completions/max_terminated_length": 5940.0,
      "completions/mean_length": 3369.0,
      "completions/mean_terminated_length": 3369.0,
      "completions/min_length": 498.0,
      "completions/min_terminated_length": 498.0,
      "entropy": 0.36095575196668506,
      "epoch": 0.22,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3427432179450989,
      "kl": 0.0036608128975785803,
      "learning_rate": 1.8848007578312686e-07,
      "loss": 0.4046,
      "num_tokens": 23825684.0,
      "reward": 0.5928125381469727,
      "reward_std": 0.5227715969085693,
      "rewards/alfworld_rollout_reward_func/mean": 0.5928125381469727,
      "rewards/alfworld_rollout_reward_func/std": 0.511535108089447,
      "sampling/importance_sampling_ratio/max": 2.6692662239074707,
      "sampling/importance_sampling_ratio/mean": 0.5836673378944397,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.313566207885742,
      "sampling/sampling_logp_difference/mean": 0.021921399980783463,
      "step": 55,
      "step_time": 209.76535455693374
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7371.0,
      "completions/max_terminated_length": 7371.0,
      "completions/mean_length": 5376.625,
      "completions/mean_terminated_length": 5376.625,
      "completions/min_length": 992.0,
      "completions/min_terminated_length": 992.0,
      "entropy": 0.335981338750571,
      "epoch": 0.224,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2279304414987564,
      "kl": 0.001375907209876459,
      "learning_rate": 1.5329319064388763e-07,
      "loss": 0.3754,
      "num_tokens": 24206844.0,
      "reward": 0.08656249940395355,
      "reward_std": 0.34256887435913086,
      "rewards/alfworld_rollout_reward_func/mean": 0.08656249940395355,
      "rewards/alfworld_rollout_reward_func/std": 0.34272125363349915,
      "sampling/importance_sampling_ratio/max": 2.9141616821289062,
      "sampling/importance_sampling_ratio/mean": 0.5004754066467285,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.75748872756958,
      "sampling/sampling_logp_difference/mean": 0.018594371154904366,
      "step": 56,
      "step_time": 289.04723902978003
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10048.0,
      "completions/max_terminated_length": 10048.0,
      "completions/mean_length": 8312.671875,
      "completions/mean_terminated_length": 8312.671875,
      "completions/min_length": 2348.0,
      "completions/min_terminated_length": 2348.0,
      "entropy": 0.3852003728970885,
      "epoch": 0.228,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.22347784042358398,
      "kl": 0.0018401142369839363,
      "learning_rate": 1.215632825696541e-07,
      "loss": 0.2461,
      "num_tokens": 24789191.0,
      "reward": 0.11437500268220901,
      "reward_std": 0.3912443518638611,
      "rewards/alfworld_rollout_reward_func/mean": 0.11437500268220901,
      "rewards/alfworld_rollout_reward_func/std": 0.403114378452301,
      "sampling/importance_sampling_ratio/max": 2.5475013256073,
      "sampling/importance_sampling_ratio/mean": 0.45833104848861694,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.6942596435546875,
      "sampling/sampling_logp_difference/mean": 0.022461190819740295,
      "step": 57,
      "step_time": 388.85974016188993
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6720.0,
      "completions/max_terminated_length": 6720.0,
      "completions/mean_length": 5543.359375,
      "completions/mean_terminated_length": 5543.359375,
      "completions/min_length": 2344.0,
      "completions/min_terminated_length": 2344.0,
      "entropy": 0.32870627008378506,
      "epoch": 0.232,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4650394320487976,
      "kl": 0.001981915997021133,
      "learning_rate": 9.337180177937954e-08,
      "loss": 0.7758,
      "num_tokens": 25182878.0,
      "reward": 0.08546875417232513,
      "reward_std": 0.2914150357246399,
      "rewards/alfworld_rollout_reward_func/mean": 0.08546875417232513,
      "rewards/alfworld_rollout_reward_func/std": 0.34556111693382263,
      "sampling/importance_sampling_ratio/max": 2.719653367996216,
      "sampling/importance_sampling_ratio/mean": 0.5657415390014648,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 5.367229461669922,
      "sampling/sampling_logp_difference/mean": 0.0186595655977726,
      "step": 58,
      "step_time": 247.89911623601802
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11541.0,
      "completions/max_terminated_length": 11541.0,
      "completions/mean_length": 5263.265625,
      "completions/mean_terminated_length": 5263.265625,
      "completions/min_length": 1418.0,
      "completions/min_terminated_length": 1418.0,
      "entropy": 0.361502249725163,
      "epoch": 0.236,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2132435142993927,
      "kl": 0.0016867622362042312,
      "learning_rate": 6.879111539992677e-08,
      "loss": 0.0722,
      "num_tokens": 25557423.0,
      "reward": 0.2345312535762787,
      "reward_std": 0.4755549430847168,
      "rewards/alfworld_rollout_reward_func/mean": 0.2345312535762787,
      "rewards/alfworld_rollout_reward_func/std": 0.486049622297287,
      "sampling/importance_sampling_ratio/max": 2.7570416927337646,
      "sampling/importance_sampling_ratio/mean": 0.4710940420627594,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.1428396701812744,
      "sampling/sampling_logp_difference/mean": 0.021931620314717293,
      "step": 59,
      "step_time": 334.8811481029552
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9499.0,
      "completions/max_terminated_length": 9499.0,
      "completions/mean_length": 5835.5625,
      "completions/mean_terminated_length": 5835.5625,
      "completions/min_length": 1881.0,
      "completions/min_terminated_length": 1881.0,
      "entropy": 0.28580303583294153,
      "epoch": 0.24,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2324402630329132,
      "kl": 0.0014885484506521607,
      "learning_rate": 4.788432170069373e-08,
      "loss": 0.0045,
      "num_tokens": 25969907.0,
      "reward": 0.2704687714576721,
      "reward_std": 0.48869654536247253,
      "rewards/alfworld_rollout_reward_func/mean": 0.27046874165534973,
      "rewards/alfworld_rollout_reward_func/std": 0.482570618391037,
      "sampling/importance_sampling_ratio/max": 2.4066574573516846,
      "sampling/importance_sampling_ratio/mean": 0.4883001446723938,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.3303287029266357,
      "sampling/sampling_logp_difference/mean": 0.016779117286205292,
      "step": 60,
      "step_time": 319.99202060801326
    }
  ],
  "logging_steps": 1,
  "max_steps": 64,
  "num_input_tokens_seen": 25969907,
  "num_train_epochs": 1,
  "save_steps": 20,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}