{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.4,
  "eval_steps": 500,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9609.0,
      "completions/max_terminated_length": 9609.0,
      "completions/mean_length": 7527.34375,
      "completions/mean_terminated_length": 7527.34375,
      "completions/min_length": 2464.0,
      "completions/min_terminated_length": 2464.0,
      "entropy": 0.31753343041054904,
      "epoch": 0.004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09699171781539917,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.1255,
      "num_tokens": 263947.0,
      "reward": 0.051750000566244125,
      "reward_std": 0.32494160532951355,
      "rewards/alfworld_rollout_reward_func/mean": 0.051750000566244125,
      "rewards/alfworld_rollout_reward_func/std": 0.3600812554359436,
      "sampling/importance_sampling_ratio/max": 1.4741449356079102,
      "sampling/importance_sampling_ratio/mean": 0.29983627796173096,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.5447624921798706,
      "sampling/sampling_logp_difference/mean": 0.018553823232650757,
      "step": 1,
      "step_time": 334.94067962296685
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9708.0,
      "completions/max_terminated_length": 9708.0,
      "completions/mean_length": 7452.15625,
      "completions/mean_terminated_length": 7452.15625,
      "completions/min_length": 1958.0,
      "completions/min_terminated_length": 1958.0,
      "entropy": 0.26579508977010846,
      "epoch": 0.008,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07386978715658188,
      "kl": 0.0,
      "learning_rate": 1.25e-06,
      "loss": 0.2055,
      "num_tokens": 528400.0,
      "reward": 0.2601250112056732,
      "reward_std": 0.5141350030899048,
      "rewards/alfworld_rollout_reward_func/mean": 0.2601250112056732,
      "rewards/alfworld_rollout_reward_func/std": 0.5191444158554077,
      "sampling/importance_sampling_ratio/max": 2.4382495880126953,
      "sampling/importance_sampling_ratio/mean": 0.5181977152824402,
      "sampling/importance_sampling_ratio/min": 0.00792621448636055,
      "sampling/sampling_logp_difference/max": 2.3046507835388184,
      "sampling/sampling_logp_difference/mean": 0.017045794054865837,
      "step": 2,
      "step_time": 321.73902505297883
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7379.0,
      "completions/max_terminated_length": 7379.0,
      "completions/mean_length": 5838.96875,
      "completions/mean_terminated_length": 5838.96875,
      "completions/min_length": 1234.0,
      "completions/min_terminated_length": 1234.0,
      "entropy": 0.405664736405015,
      "epoch": 0.012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.10100319981575012,
      "kl": 0.0017308223832515068,
      "learning_rate": 2.5e-06,
      "loss": -0.079,
      "num_tokens": 734703.0,
      "reward": 0.140687495470047,
      "reward_std": 0.4871714115142822,
      "rewards/alfworld_rollout_reward_func/mean": 0.140687495470047,
      "rewards/alfworld_rollout_reward_func/std": 0.47618311643600464,
      "sampling/importance_sampling_ratio/max": 2.0904979705810547,
      "sampling/importance_sampling_ratio/mean": 0.32961514592170715,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.584744930267334,
      "sampling/sampling_logp_difference/mean": 0.022431109100580215,
      "step": 3,
      "step_time": 272.60163002300396
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10150.0,
      "completions/max_terminated_length": 10150.0,
      "completions/mean_length": 8443.78125,
      "completions/mean_terminated_length": 8443.78125,
      "completions/min_length": 5169.0,
      "completions/min_terminated_length": 5169.0,
      "entropy": 0.3736097775399685,
      "epoch": 0.016,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.057365067303180695,
      "kl": 0.0021683564627892338,
      "learning_rate": 3.7500000000000005e-06,
      "loss": -0.1302,
      "num_tokens": 1027944.0,
      "reward": -0.03568749502301216,
      "reward_std": 0.16881901025772095,
      "rewards/alfworld_rollout_reward_func/mean": -0.03568749502301216,
      "rewards/alfworld_rollout_reward_func/std": 0.27729666233062744,
      "sampling/importance_sampling_ratio/max": 0.9850316047668457,
      "sampling/importance_sampling_ratio/mean": 0.23796725273132324,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0175373554229736,
      "sampling/sampling_logp_difference/mean": 0.022764241322875023,
      "step": 4,
      "step_time": 360.9181504939843
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5416.0,
      "completions/max_terminated_length": 5416.0,
      "completions/mean_length": 4855.84375,
      "completions/mean_terminated_length": 4855.84375,
      "completions/min_length": 4281.0,
      "completions/min_terminated_length": 4281.0,
      "entropy": 0.3696938706561923,
      "epoch": 0.02,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.16541753709316254,
      "kl": 0.001998385167098604,
      "learning_rate": 5e-06,
      "loss": -0.3616,
      "num_tokens": 1199523.0,
      "reward": -0.09624999761581421,
      "reward_std": 0.046621114015579224,
      "rewards/alfworld_rollout_reward_func/mean": -0.09624999761581421,
      "rewards/alfworld_rollout_reward_func/std": 0.04897332936525345,
      "sampling/importance_sampling_ratio/max": 2.4984867572784424,
      "sampling/importance_sampling_ratio/mean": 0.5033700466156006,
      "sampling/importance_sampling_ratio/min": 0.00037635909393429756,
      "sampling/sampling_logp_difference/max": 3.5292906761169434,
      "sampling/sampling_logp_difference/mean": 0.022726019844412804,
      "step": 5,
      "step_time": 238.88000224298594
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11870.0,
      "completions/max_terminated_length": 11870.0,
      "completions/mean_length": 9758.875,
      "completions/mean_terminated_length": 9758.875,
      "completions/min_length": 2533.0,
      "completions/min_terminated_length": 2533.0,
      "entropy": 0.4001495079137385,
      "epoch": 0.024,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06449950486421585,
      "kl": 0.0019289798929094104,
      "learning_rate": 4.999157413258782e-06,
      "loss": 0.0376,
      "num_tokens": 1539231.0,
      "reward": 0.030937502160668373,
      "reward_std": 0.336625874042511,
      "rewards/alfworld_rollout_reward_func/mean": 0.030937502160668373,
      "rewards/alfworld_rollout_reward_func/std": 0.37901195883750916,
      "sampling/importance_sampling_ratio/max": 1.6886974573135376,
      "sampling/importance_sampling_ratio/mean": 0.3790336847305298,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.1409363746643066,
      "sampling/sampling_logp_difference/mean": 0.023546157404780388,
      "step": 6,
      "step_time": 440.73640958298347
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9966.0,
      "completions/max_terminated_length": 9966.0,
      "completions/mean_length": 8191.53125,
      "completions/mean_terminated_length": 8191.53125,
      "completions/min_length": 7362.0,
      "completions/min_terminated_length": 7362.0,
      "entropy": 0.32905709464102983,
      "epoch": 0.028,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.04034750163555145,
      "kl": 0.0018940924492198974,
      "learning_rate": 4.996630220997058e-06,
      "loss": -0.1075,
      "num_tokens": 1824368.0,
      "reward": -0.049187496304512024,
      "reward_std": 0.12530304491519928,
      "rewards/alfworld_rollout_reward_func/mean": -0.049187496304512024,
      "rewards/alfworld_rollout_reward_func/std": 0.1951422095298767,
      "sampling/importance_sampling_ratio/max": 2.443614959716797,
      "sampling/importance_sampling_ratio/mean": 0.3964727818965912,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.6266517639160156,
      "sampling/sampling_logp_difference/mean": 0.020559251308441162,
      "step": 7,
      "step_time": 350.02132217200415
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5095.0,
      "completions/max_terminated_length": 5095.0,
      "completions/mean_length": 4314.65625,
      "completions/mean_terminated_length": 4314.65625,
      "completions/min_length": 736.0,
      "completions/min_terminated_length": 736.0,
      "entropy": 0.3601735474076122,
      "epoch": 0.032,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08598003536462784,
      "kl": 0.0017114684615080478,
      "learning_rate": 4.992420126717784e-06,
      "loss": 0.2628,
      "num_tokens": 1978501.0,
      "reward": 0.09481249749660492,
      "reward_std": 0.3786434829235077,
      "rewards/alfworld_rollout_reward_func/mean": 0.09481249749660492,
      "rewards/alfworld_rollout_reward_func/std": 0.430143803358078,
      "sampling/importance_sampling_ratio/max": 2.776649236679077,
      "sampling/importance_sampling_ratio/mean": 0.47567594051361084,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.7654576301574707,
      "sampling/sampling_logp_difference/mean": 0.019868386909365654,
      "step": 8,
      "step_time": 225.1138117579976
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9537.0,
      "completions/max_terminated_length": 9537.0,
      "completions/mean_length": 7083.96875,
      "completions/mean_terminated_length": 7083.96875,
      "completions/min_length": 1562.0,
      "completions/min_terminated_length": 1562.0,
      "entropy": 0.30733060790225863,
      "epoch": 0.036,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09293514490127563,
      "kl": 0.0028748424542754947,
      "learning_rate": 4.986529968316654e-06,
      "loss": 0.2734,
      "num_tokens": 2231140.0,
      "reward": 0.3280624747276306,
      "reward_std": 0.5648518204689026,
      "rewards/alfworld_rollout_reward_func/mean": 0.3280624747276306,
      "rewards/alfworld_rollout_reward_func/std": 0.5717496871948242,
      "sampling/importance_sampling_ratio/max": 2.961937665939331,
      "sampling/importance_sampling_ratio/mean": 0.5576246380805969,
      "sampling/importance_sampling_ratio/min": 0.05429670959711075,
      "sampling/sampling_logp_difference/max": 2.148742198944092,
      "sampling/sampling_logp_difference/mean": 0.019244220107793808,
      "step": 9,
      "step_time": 323.0066561399726
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9496.0,
      "completions/max_terminated_length": 9496.0,
      "completions/mean_length": 7961.59375,
      "completions/mean_terminated_length": 7961.59375,
      "completions/min_length": 7039.0,
      "completions/min_terminated_length": 7039.0,
      "entropy": 0.3252652743831277,
      "epoch": 0.04,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.05244140699505806,
      "kl": 0.002839431928805425,
      "learning_rate": 4.978963716169166e-06,
      "loss": -0.1758,
      "num_tokens": 2508535.0,
      "reward": -0.0949999988079071,
      "reward_std": 0.034666046500205994,
      "rewards/alfworld_rollout_reward_func/mean": -0.0949999988079071,
      "rewards/alfworld_rollout_reward_func/std": 0.04079848527908325,
      "sampling/importance_sampling_ratio/max": 1.3879799842834473,
      "sampling/importance_sampling_ratio/mean": 0.34057629108428955,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 47.49563217163086,
      "sampling/sampling_logp_difference/mean": 0.027333300560712814,
      "step": 10,
      "step_time": 328.04954918399017
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5477.0,
      "completions/max_terminated_length": 5477.0,
      "completions/mean_length": 5025.0,
      "completions/mean_terminated_length": 5025.0,
      "completions/min_length": 3290.0,
      "completions/min_terminated_length": 3290.0,
      "entropy": 0.37359505658969283,
      "epoch": 0.044,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07209861278533936,
      "kl": 0.0015697149883635575,
      "learning_rate": 4.9697264704543135e-06,
      "loss": 0.0374,
      "num_tokens": 2686583.0,
      "reward": -0.0533750019967556,
      "reward_std": 0.12613213062286377,
      "rewards/alfworld_rollout_reward_func/mean": -0.0533750019967556,
      "rewards/alfworld_rollout_reward_func/std": 0.19394871592521667,
      "sampling/importance_sampling_ratio/max": 2.769270658493042,
      "sampling/importance_sampling_ratio/mean": 0.48209959268569946,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 54.623374938964844,
      "sampling/sampling_logp_difference/mean": 0.02327369526028633,
      "step": 11,
      "step_time": 223.6848387320133
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9316.0,
      "completions/max_terminated_length": 9316.0,
      "completions/mean_length": 8667.5,
      "completions/mean_terminated_length": 8667.5,
      "completions/min_length": 8263.0,
      "completions/min_terminated_length": 8263.0,
      "entropy": 0.29008231218904257,
      "epoch": 0.048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.036121197044849396,
      "kl": 0.0016753725431044586,
      "learning_rate": 4.958824457716707e-06,
      "loss": -0.0967,
      "num_tokens": 2988551.0,
      "reward": -0.08687499910593033,
      "reward_std": 0.043158818036317825,
      "rewards/alfworld_rollout_reward_func/mean": -0.08687499910593033,
      "rewards/alfworld_rollout_reward_func/std": 0.046102095395326614,
      "sampling/importance_sampling_ratio/max": 1.5300953388214111,
      "sampling/importance_sampling_ratio/mean": 0.3308807611465454,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7159793376922607,
      "sampling/sampling_logp_difference/mean": 0.018939001485705376,
      "step": 12,
      "step_time": 356.27002431899746
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9878.0,
      "completions/max_terminated_length": 9878.0,
      "completions/mean_length": 8581.75,
      "completions/mean_terminated_length": 8581.75,
      "completions/min_length": 7974.0,
      "completions/min_terminated_length": 7974.0,
      "entropy": 0.2544857235625386,
      "epoch": 0.052,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12804464995861053,
      "kl": 0.0013123316457495093,
      "learning_rate": 4.9462650266694546e-06,
      "loss": -0.2577,
      "num_tokens": 3287647.0,
      "reward": -0.02643749862909317,
      "reward_std": 0.12068305909633636,
      "rewards/alfworld_rollout_reward_func/mean": -0.02643749862909317,
      "rewards/alfworld_rollout_reward_func/std": 0.18352726101875305,
      "sampling/importance_sampling_ratio/max": 2.085613965988159,
      "sampling/importance_sampling_ratio/mean": 0.6230069398880005,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 5.174288749694824,
      "sampling/sampling_logp_difference/mean": 0.01557723619043827,
      "step": 13,
      "step_time": 371.6143698649903
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5113.0,
      "completions/max_terminated_length": 5113.0,
      "completions/mean_length": 3425.375,
      "completions/mean_terminated_length": 3425.375,
      "completions/min_length": 1258.0,
      "completions/min_terminated_length": 1258.0,
      "entropy": 0.3549840818159282,
      "epoch": 0.056,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06998535245656967,
      "kl": 0.0019005965677934,
      "learning_rate": 4.9320566432406185e-06,
      "loss": 0.0376,
      "num_tokens": 3412747.0,
      "reward": 0.4362499713897705,
      "reward_std": 0.5676650404930115,
      "rewards/alfworld_rollout_reward_func/mean": 0.4362499713897705,
      "rewards/alfworld_rollout_reward_func/std": 0.5570400357246399,
      "sampling/importance_sampling_ratio/max": 2.0355820655822754,
      "sampling/importance_sampling_ratio/mean": 0.41046884655952454,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.2833807468414307,
      "sampling/sampling_logp_difference/mean": 0.02138603664934635,
      "step": 14,
      "step_time": 175.46163126500323
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9538.0,
      "completions/max_terminated_length": 9538.0,
      "completions/mean_length": 8678.4375,
      "completions/mean_terminated_length": 8678.4375,
      "completions/min_length": 8210.0,
      "completions/min_terminated_length": 8210.0,
      "entropy": 0.2878416678868234,
      "epoch": 0.06,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.03550809249281883,
      "kl": 0.0012492626774474047,
      "learning_rate": 4.916208884866593e-06,
      "loss": 0.0712,
      "num_tokens": 3714873.0,
      "reward": -0.07062500715255737,
      "reward_std": 0.04733540862798691,
      "rewards/alfworld_rollout_reward_func/mean": -0.07062500715255737,
      "rewards/alfworld_rollout_reward_func/std": 0.04696859046816826,
      "sampling/importance_sampling_ratio/max": 1.1348778009414673,
      "sampling/importance_sampling_ratio/mean": 0.31592193245887756,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4374537467956543,
      "sampling/sampling_logp_difference/mean": 0.017727136611938477,
      "step": 15,
      "step_time": 364.41606586901617
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5416.0,
      "completions/max_terminated_length": 5416.0,
      "completions/mean_length": 1625.96875,
      "completions/mean_terminated_length": 1625.96875,
      "completions/min_length": 705.0,
      "completions/min_terminated_length": 705.0,
      "entropy": 0.27994309086352587,
      "epoch": 0.064,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.023568756878376007,
      "kl": 0.0017632305944061955,
      "learning_rate": 4.8987324340362445e-06,
      "loss": -0.0524,
      "num_tokens": 3782456.0,
      "reward": 0.8765000104904175,
      "reward_std": 0.3589959144592285,
      "rewards/alfworld_rollout_reward_func/mean": 0.8765000104904175,
      "rewards/alfworld_rollout_reward_func/std": 0.39898157119750977,
      "sampling/importance_sampling_ratio/max": 1.7719885110855103,
      "sampling/importance_sampling_ratio/mean": 0.5624613165855408,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.212910532951355,
      "sampling/sampling_logp_difference/mean": 0.021020911633968353,
      "step": 16,
      "step_time": 121.96809069301526
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5848.0,
      "completions/max_terminated_length": 5848.0,
      "completions/mean_length": 5169.84375,
      "completions/mean_terminated_length": 5169.84375,
      "completions/min_length": 3424.0,
      "completions/min_terminated_length": 3424.0,
      "entropy": 0.38696050783619285,
      "epoch": 0.068,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09861446171998978,
      "kl": 0.0013451038357743528,
      "learning_rate": 4.879639071090174e-06,
      "loss": 0.1571,
      "num_tokens": 3965683.0,
      "reward": -0.02056249976158142,
      "reward_std": 0.22134613990783691,
      "rewards/alfworld_rollout_reward_func/mean": -0.02056249976158142,
      "rewards/alfworld_rollout_reward_func/std": 0.2705713212490082,
      "sampling/importance_sampling_ratio/max": 2.602855682373047,
      "sampling/importance_sampling_ratio/mean": 0.4220735430717468,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9502549171447754,
      "sampling/sampling_logp_difference/mean": 0.02037898264825344,
      "step": 17,
      "step_time": 239.7622231810019
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 12576.0,
      "completions/max_terminated_length": 12576.0,
      "completions/mean_length": 11056.15625,
      "completions/mean_terminated_length": 11056.15625,
      "completions/min_length": 4799.0,
      "completions/min_terminated_length": 4799.0,
      "entropy": 0.4063271093182266,
      "epoch": 0.072,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06012088432908058,
      "kl": 0.0020217235087329755,
      "learning_rate": 4.858941666279956e-06,
      "loss": 0.0168,
      "num_tokens": 4347448.0,
      "reward": -0.08149999380111694,
      "reward_std": 0.13932166993618011,
      "rewards/alfworld_rollout_reward_func/mean": -0.08149999380111694,
      "rewards/alfworld_rollout_reward_func/std": 0.20591196417808533,
      "sampling/importance_sampling_ratio/max": 2.379767894744873,
      "sampling/importance_sampling_ratio/mean": 0.33124375343322754,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7336961030960083,
      "sampling/sampling_logp_difference/mean": 0.023929107934236526,
      "step": 18,
      "step_time": 465.16173633500875
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6323.0,
      "completions/max_terminated_length": 6323.0,
      "completions/mean_length": 4971.09375,
      "completions/mean_terminated_length": 4971.09375,
      "completions/min_length": 3896.0,
      "completions/min_terminated_length": 3896.0,
      "entropy": 0.4407036453485489,
      "epoch": 0.076,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1755954921245575,
      "kl": 0.0018943681934615597,
      "learning_rate": 4.836654171092683e-06,
      "loss": -0.3381,
      "num_tokens": 4522491.0,
      "reward": -0.022624999284744263,
      "reward_std": 0.20558400452136993,
      "rewards/alfworld_rollout_reward_func/mean": -0.022624999284744263,
      "rewards/alfworld_rollout_reward_func/std": 0.26010042428970337,
      "sampling/importance_sampling_ratio/max": 2.1797361373901367,
      "sampling/importance_sampling_ratio/mean": 0.3776967525482178,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9502511024475098,
      "sampling/sampling_logp_difference/mean": 0.02467840164899826,
      "step": 19,
      "step_time": 243.88053144400328
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11207.0,
      "completions/max_terminated_length": 11207.0,
      "completions/mean_length": 9984.3125,
      "completions/mean_terminated_length": 9984.3125,
      "completions/min_length": 4179.0,
      "completions/min_terminated_length": 4179.0,
      "entropy": 0.3350094626657665,
      "epoch": 0.08,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0771276131272316,
      "kl": 0.0018198210955233662,
      "learning_rate": 4.812791608846709e-06,
      "loss": 0.0206,
      "num_tokens": 4869989.0,
      "reward": 0.09581249952316284,
      "reward_std": 0.44819241762161255,
      "rewards/alfworld_rollout_reward_func/mean": 0.09581249952316284,
      "rewards/alfworld_rollout_reward_func/std": 0.43458762764930725,
      "sampling/importance_sampling_ratio/max": 1.8481526374816895,
      "sampling/importance_sampling_ratio/mean": 0.38451921939849854,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.429098129272461,
      "sampling/sampling_logp_difference/mean": 0.020724384114146233,
      "step": 20,
      "step_time": 405.4802599009927
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5980.0,
      "completions/max_terminated_length": 5980.0,
      "completions/mean_length": 2868.5,
      "completions/mean_terminated_length": 2868.5,
      "completions/min_length": 623.0,
      "completions/min_terminated_length": 623.0,
      "entropy": 0.3131830710917711,
      "epoch": 0.084,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.044104401022195816,
      "kl": 0.0020016076996398624,
      "learning_rate": 4.787370064564884e-06,
      "loss": -0.0577,
      "num_tokens": 4979029.0,
      "reward": 0.6869999766349792,
      "reward_std": 0.5118247866630554,
      "rewards/alfworld_rollout_reward_func/mean": 0.6869999766349792,
      "rewards/alfworld_rollout_reward_func/std": 0.5184879302978516,
      "sampling/importance_sampling_ratio/max": 2.4770655632019043,
      "sampling/importance_sampling_ratio/mean": 0.6952197551727295,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0350642204284668,
      "sampling/sampling_logp_difference/mean": 0.020318225026130676,
      "step": 21,
      "step_time": 161.65337034598633
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8776.0,
      "completions/max_terminated_length": 8776.0,
      "completions/mean_length": 6767.40625,
      "completions/mean_terminated_length": 6767.40625,
      "completions/min_length": 2421.0,
      "completions/min_terminated_length": 2421.0,
      "entropy": 0.35168757336214185,
      "epoch": 0.088,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09959151595830917,
      "kl": 0.0017227518037543632,
      "learning_rate": 4.760406674132126e-06,
      "loss": -0.0616,
      "num_tokens": 5216002.0,
      "reward": 0.18024998903274536,
      "reward_std": 0.5159989595413208,
      "rewards/alfworld_rollout_reward_func/mean": 0.18024998903274536,
      "rewards/alfworld_rollout_reward_func/std": 0.5085508227348328,
      "sampling/importance_sampling_ratio/max": 2.815340518951416,
      "sampling/importance_sampling_ratio/mean": 0.5774703621864319,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.2189407348632812,
      "sampling/sampling_logp_difference/mean": 0.02067497931420803,
      "step": 22,
      "step_time": 312.3654504569786
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5316.0,
      "completions/max_terminated_length": 5316.0,
      "completions/mean_length": 4200.6875,
      "completions/mean_terminated_length": 4200.6875,
      "completions/min_length": 526.0,
      "completions/min_terminated_length": 526.0,
      "entropy": 0.3291297680698335,
      "epoch": 0.092,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12509657442569733,
      "kl": 0.0014415504001590307,
      "learning_rate": 4.73191961274466e-06,
      "loss": 0.0168,
      "num_tokens": 5367672.0,
      "reward": 0.250249981880188,
      "reward_std": 0.49357783794403076,
      "rewards/alfworld_rollout_reward_func/mean": 0.250249981880188,
      "rewards/alfworld_rollout_reward_func/std": 0.5176389813423157,
      "sampling/importance_sampling_ratio/max": 2.6394360065460205,
      "sampling/importance_sampling_ratio/mean": 0.485737681388855,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9740701913833618,
      "sampling/sampling_logp_difference/mean": 0.018988091498613358,
      "step": 23,
      "step_time": 205.61723882200022
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5118.0,
      "completions/max_terminated_length": 5118.0,
      "completions/mean_length": 3482.15625,
      "completions/mean_terminated_length": 3482.15625,
      "completions/min_length": 1306.0,
      "completions/min_terminated_length": 1306.0,
      "entropy": 0.3946748194284737,
      "epoch": 0.096,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08480077236890793,
      "kl": 0.0012077928076905664,
      "learning_rate": 4.701928082658661e-06,
      "loss": 0.1749,
      "num_tokens": 5495421.0,
      "reward": 0.5172500014305115,
      "reward_std": 0.5834309458732605,
      "rewards/alfworld_rollout_reward_func/mean": 0.5172500014305115,
      "rewards/alfworld_rollout_reward_func/std": 0.5709651708602905,
      "sampling/importance_sampling_ratio/max": 2.6909029483795166,
      "sampling/importance_sampling_ratio/mean": 0.5612046718597412,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.8878040313720703,
      "sampling/sampling_logp_difference/mean": 0.020074568688869476,
      "step": 24,
      "step_time": 182.43979760597722
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8626.0,
      "completions/max_terminated_length": 8626.0,
      "completions/mean_length": 7605.125,
      "completions/mean_terminated_length": 7605.125,
      "completions/min_length": 4181.0,
      "completions/min_terminated_length": 4181.0,
      "entropy": 0.34247989440336823,
      "epoch": 0.1,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.117367684841156,
      "kl": 0.0018653218503459357,
      "learning_rate": 4.67045230024661e-06,
      "loss": -0.4289,
      "num_tokens": 5760993.0,
      "reward": -0.047874994575977325,
      "reward_std": 0.13079281151294708,
      "rewards/alfworld_rollout_reward_func/mean": -0.047874994575977325,
      "rewards/alfworld_rollout_reward_func/std": 0.1973174661397934,
      "sampling/importance_sampling_ratio/max": 2.7526066303253174,
      "sampling/importance_sampling_ratio/mean": 0.5172563195228577,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.884810447692871,
      "sampling/sampling_logp_difference/mean": 0.020854920148849487,
      "step": 25,
      "step_time": 318.9381142439961
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11381.0,
      "completions/max_terminated_length": 11381.0,
      "completions/mean_length": 9974.5,
      "completions/mean_terminated_length": 9974.5,
      "completions/min_length": 9364.0,
      "completions/min_terminated_length": 9364.0,
      "entropy": 0.3294394067488611,
      "epoch": 0.104,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0581807978451252,
      "kl": 0.0015963109199219616,
      "learning_rate": 4.6375134823700505e-06,
      "loss": 0.0943,
      "num_tokens": 6107313.0,
      "reward": -0.06718749552965164,
      "reward_std": 0.10619112849235535,
      "rewards/alfworld_rollout_reward_func/mean": -0.06718749552965164,
      "rewards/alfworld_rollout_reward_func/std": 0.17335838079452515,
      "sampling/importance_sampling_ratio/max": 2.373974084854126,
      "sampling/importance_sampling_ratio/mean": 0.4386903941631317,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.610280990600586,
      "sampling/sampling_logp_difference/mean": 0.01913020759820938,
      "step": 26,
      "step_time": 414.68458038300014
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6944.0,
      "completions/max_terminated_length": 6944.0,
      "completions/mean_length": 6449.375,
      "completions/mean_terminated_length": 6449.375,
      "completions/min_length": 5182.0,
      "completions/min_terminated_length": 5182.0,
      "entropy": 0.290782836265862,
      "epoch": 0.108,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1415673792362213,
      "kl": 0.0014344851497298805,
      "learning_rate": 4.603133832077953e-06,
      "loss": 0.3656,
      "num_tokens": 6334429.0,
      "reward": 0.0663750022649765,
      "reward_std": 0.2861044108867645,
      "rewards/alfworld_rollout_reward_func/mean": 0.0663750022649765,
      "rewards/alfworld_rollout_reward_func/std": 0.4014340043067932,
      "sampling/importance_sampling_ratio/max": 2.9577462673187256,
      "sampling/importance_sampling_ratio/mean": 0.4735347628593445,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.694256067276001,
      "sampling/sampling_logp_difference/mean": 0.017661554738879204,
      "step": 27,
      "step_time": 254.20073604799836
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4959.0,
      "completions/max_terminated_length": 4959.0,
      "completions/mean_length": 2397.875,
      "completions/mean_terminated_length": 2397.875,
      "completions/min_length": 519.0,
      "completions/min_terminated_length": 519.0,
      "entropy": 0.34110196540132165,
      "epoch": 0.112,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07966114580631256,
      "kl": 0.0017357735669065733,
      "learning_rate": 4.567336523640322e-06,
      "loss": 0.0816,
      "num_tokens": 6426137.0,
      "reward": 0.7531875371932983,
      "reward_std": 0.42287677526474,
      "rewards/alfworld_rollout_reward_func/mean": 0.7531875371932983,
      "rewards/alfworld_rollout_reward_func/std": 0.48645731806755066,
      "sampling/importance_sampling_ratio/max": 2.248128890991211,
      "sampling/importance_sampling_ratio/mean": 0.5701067447662354,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0523573160171509,
      "sampling/sampling_logp_difference/mean": 0.020480383187532425,
      "step": 28,
      "step_time": 151.82388260698644
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9672.0,
      "completions/max_terminated_length": 9672.0,
      "completions/mean_length": 8832.34375,
      "completions/mean_terminated_length": 8832.34375,
      "completions/min_length": 2991.0,
      "completions/min_terminated_length": 2991.0,
      "entropy": 0.32463116594590247,
      "epoch": 0.116,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.055072441697120667,
      "kl": 0.0014255273326853057,
      "learning_rate": 4.530145686927126e-06,
      "loss": 0.1313,
      "num_tokens": 6734084.0,
      "reward": -0.022187499329447746,
      "reward_std": 0.213197261095047,
      "rewards/alfworld_rollout_reward_func/mean": -0.022187499329447746,
      "rewards/alfworld_rollout_reward_func/std": 0.2705041170120239,
      "sampling/importance_sampling_ratio/max": 2.71278715133667,
      "sampling/importance_sampling_ratio/mean": 0.5353480577468872,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.212910532951355,
      "sampling/sampling_logp_difference/mean": 0.019132211804389954,
      "step": 29,
      "step_time": 386.17356162999204
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10229.0,
      "completions/max_terminated_length": 10229.0,
      "completions/mean_length": 7089.03125,
      "completions/mean_terminated_length": 7089.03125,
      "completions/min_length": 2106.0,
      "completions/min_terminated_length": 2106.0,
      "entropy": 0.34196448139846325,
      "epoch": 0.12,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0913311019539833,
      "kl": 0.0014697671704198,
      "learning_rate": 4.49158639114309e-06,
      "loss": 0.1673,
      "num_tokens": 6986309.0,
      "reward": 0.3622500002384186,
      "reward_std": 0.5987363457679749,
      "rewards/alfworld_rollout_reward_func/mean": 0.3622500002384186,
      "rewards/alfworld_rollout_reward_func/std": 0.5764731168746948,
      "sampling/importance_sampling_ratio/max": 2.353553533554077,
      "sampling/importance_sampling_ratio/mean": 0.5580101013183594,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 55.49094009399414,
      "sampling/sampling_logp_difference/mean": 0.02466428652405739,
      "step": 30,
      "step_time": 361.2400383850036
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7853.0,
      "completions/max_terminated_length": 7853.0,
      "completions/mean_length": 6668.875,
      "completions/mean_terminated_length": 6668.875,
      "completions/min_length": 1438.0,
      "completions/min_terminated_length": 1438.0,
      "entropy": 0.25103138061240315,
      "epoch": 0.124,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12625202536582947,
      "kl": 0.0013389530686254147,
      "learning_rate": 4.45168462792932e-06,
      "loss": -0.3054,
      "num_tokens": 7220673.0,
      "reward": 0.08543749898672104,
      "reward_std": 0.33267462253570557,
      "rewards/alfworld_rollout_reward_func/mean": 0.08543749898672104,
      "rewards/alfworld_rollout_reward_func/std": 0.3945035934448242,
      "sampling/importance_sampling_ratio/max": 2.542257308959961,
      "sampling/importance_sampling_ratio/mean": 0.5212557315826416,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7397236824035645,
      "sampling/sampling_logp_difference/mean": 0.014140520244836807,
      "step": 31,
      "step_time": 298.24992308700166
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 13314.0,
      "completions/max_terminated_length": 13314.0,
      "completions/mean_length": 10635.21875,
      "completions/mean_terminated_length": 10635.21875,
      "completions/min_length": 2254.0,
      "completions/min_terminated_length": 2254.0,
      "entropy": 0.327601904515177,
      "epoch": 0.128,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07485339790582657,
      "kl": 0.0022537091454069014,
      "learning_rate": 4.410467293843123e-06,
      "loss": 0.0829,
      "num_tokens": 7592744.0,
      "reward": 0.15349999070167542,
      "reward_std": 0.4711422920227051,
      "rewards/alfworld_rollout_reward_func/mean": 0.15349999070167542,
      "rewards/alfworld_rollout_reward_func/std": 0.4708044230937958,
      "sampling/importance_sampling_ratio/max": 2.957587480545044,
      "sampling/importance_sampling_ratio/mean": 0.6460099220275879,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.6615216732025146,
      "sampling/sampling_logp_difference/mean": 0.0228976272046566,
      "step": 32,
      "step_time": 485.91699508898455
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5669.0,
      "completions/max_terminated_length": 5669.0,
      "completions/mean_length": 5010.8125,
      "completions/mean_terminated_length": 5010.8125,
      "completions/min_length": 4272.0,
      "completions/min_terminated_length": 4272.0,
      "entropy": 0.3313480073120445,
      "epoch": 0.132,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.11383697390556335,
      "kl": 0.002066290511720581,
      "learning_rate": 4.367962172227866e-06,
      "loss": 0.2319,
      "num_tokens": 7770434.0,
      "reward": -0.043312497437000275,
      "reward_std": 0.21143738925457,
      "rewards/alfworld_rollout_reward_func/mean": -0.043312497437000275,
      "rewards/alfworld_rollout_reward_func/std": 0.2677001953125,
      "sampling/importance_sampling_ratio/max": 1.8964158296585083,
      "sampling/importance_sampling_ratio/mean": 0.4280845522880554,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 48.41267395019531,
      "sampling/sampling_logp_difference/mean": 0.026195591315627098,
      "step": 33,
      "step_time": 216.44529388601222
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9870.0,
      "completions/max_terminated_length": 9870.0,
      "completions/mean_length": 7890.9375,
      "completions/mean_terminated_length": 7890.9375,
      "completions/min_length": 1615.0,
      "completions/min_terminated_length": 1615.0,
      "entropy": 0.3045849180780351,
      "epoch": 0.136,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06688809394836426,
      "kl": 0.001580554380780086,
      "learning_rate": 4.324197914485075e-06,
      "loss": 0.1723,
      "num_tokens": 8048832.0,
      "reward": 0.18268749117851257,
      "reward_std": 0.5357717871665955,
      "rewards/alfworld_rollout_reward_func/mean": 0.18268749117851257,
      "rewards/alfworld_rollout_reward_func/std": 0.5283057689666748,
      "sampling/importance_sampling_ratio/max": 1.4296221733093262,
      "sampling/importance_sampling_ratio/mean": 0.4630698561668396,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 57.18749237060547,
      "sampling/sampling_logp_difference/mean": 0.0222089234739542,
      "step": 34,
      "step_time": 366.9164286329906
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10091.0,
      "completions/max_terminated_length": 10091.0,
      "completions/mean_length": 6937.8125,
      "completions/mean_terminated_length": 6937.8125,
      "completions/min_length": 2364.0,
      "completions/min_terminated_length": 2364.0,
      "entropy": 0.3189437501132488,
      "epoch": 0.14,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06047705188393593,
      "kl": 0.0017184474909299752,
      "learning_rate": 4.279204020761401e-06,
      "loss": 0.0459,
      "num_tokens": 8296602.0,
      "reward": 0.35374999046325684,
      "reward_std": 0.5226367712020874,
      "rewards/alfworld_rollout_reward_func/mean": 0.35374999046325684,
      "rewards/alfworld_rollout_reward_func/std": 0.5893219113349915,
      "sampling/importance_sampling_ratio/max": 2.7987327575683594,
      "sampling/importance_sampling_ratio/mean": 0.5202946662902832,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 56.38393020629883,
      "sampling/sampling_logp_difference/mean": 0.02321350947022438,
      "step": 35,
      "step_time": 352.48793815699173
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6439.0,
      "completions/max_terminated_length": 6439.0,
      "completions/mean_length": 5805.8125,
      "completions/mean_terminated_length": 5805.8125,
      "completions/min_length": 5410.0,
      "completions/min_terminated_length": 5410.0,
      "entropy": 0.31256520189344883,
      "epoch": 0.144,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06142375245690346,
      "kl": 0.0011960903293584124,
      "learning_rate": 4.233010820063473e-06,
      "loss": -0.0928,
      "num_tokens": 8501748.0,
      "reward": -0.07906249910593033,
      "reward_std": 0.03072059527039528,
      "rewards/alfworld_rollout_reward_func/mean": -0.07906249910593033,
      "rewards/alfworld_rollout_reward_func/std": 0.031862542033195496,
      "sampling/importance_sampling_ratio/max": 1.3057942390441895,
      "sampling/importance_sampling_ratio/mean": 0.35130757093429565,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0284595489501953,
      "sampling/sampling_logp_difference/mean": 0.017173780128359795,
      "step": 36,
      "step_time": 262.96477444199263
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10178.0,
      "completions/max_terminated_length": 10178.0,
      "completions/mean_length": 8620.375,
      "completions/mean_terminated_length": 8620.375,
      "completions/min_length": 2023.0,
      "completions/min_terminated_length": 2023.0,
      "entropy": 0.3134400031995028,
      "epoch": 0.148,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.047842346131801605,
      "kl": 0.001771117205862538,
      "learning_rate": 4.185649449814046e-06,
      "loss": 0.0814,
      "num_tokens": 8803296.0,
      "reward": 0.016124999150633812,
      "reward_std": 0.34558093547821045,
      "rewards/alfworld_rollout_reward_func/mean": 0.016124999150633812,
      "rewards/alfworld_rollout_reward_func/std": 0.3839482069015503,
      "sampling/importance_sampling_ratio/max": 2.2736334800720215,
      "sampling/importance_sampling_ratio/mean": 0.3046373724937439,
      "sampling/importance_sampling_ratio/min": 8.543683307914326e-28,
      "sampling/sampling_logp_difference/max": 54.35212707519531,
      "sampling/sampling_logp_difference/mean": 0.02278374321758747,
      "step": 37,
      "step_time": 377.8420012939969
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5236.0,
      "completions/max_terminated_length": 5236.0,
      "completions/mean_length": 4332.15625,
      "completions/mean_terminated_length": 4332.15625,
      "completions/min_length": 557.0,
      "completions/min_terminated_length": 557.0,
      "entropy": 0.39184045884758234,
      "epoch": 0.152,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13184601068496704,
      "kl": 0.0018992484310729196,
      "learning_rate": 4.137151834863213e-06,
      "loss": 0.0845,
      "num_tokens": 8958853.0,
      "reward": 0.10606250166893005,
      "reward_std": 0.37507200241088867,
      "rewards/alfworld_rollout_reward_func/mean": 0.10606250166893005,
      "rewards/alfworld_rollout_reward_func/std": 0.44086700677871704,
      "sampling/importance_sampling_ratio/max": 2.8395280838012695,
      "sampling/importance_sampling_ratio/mean": 0.7063838243484497,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 55.17744445800781,
      "sampling/sampling_logp_difference/mean": 0.02787482924759388,
      "step": 38,
      "step_time": 216.3319418339961
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6895.0,
      "completions/max_terminated_length": 6895.0,
      "completions/mean_length": 5100.90625,
      "completions/mean_terminated_length": 5100.90625,
      "completions/min_length": 1905.0,
      "completions/min_terminated_length": 1905.0,
      "entropy": 0.3214424313046038,
      "epoch": 0.156,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08944570273160934,
      "kl": 0.0015830010088393465,
      "learning_rate": 4.087550665968846e-06,
      "loss": 0.2,
      "num_tokens": 9141858.0,
      "reward": 0.32487499713897705,
      "reward_std": 0.5901329517364502,
      "rewards/alfworld_rollout_reward_func/mean": 0.32487499713897705,
      "rewards/alfworld_rollout_reward_func/std": 0.5753059983253479,
      "sampling/importance_sampling_ratio/max": 2.313490867614746,
      "sampling/importance_sampling_ratio/mean": 0.576886773109436,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0487456321716309,
      "sampling/sampling_logp_difference/mean": 0.019821226596832275,
      "step": 39,
      "step_time": 252.95276514600118
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6532.0,
      "completions/max_terminated_length": 6532.0,
      "completions/mean_length": 5208.625,
      "completions/mean_terminated_length": 5208.625,
      "completions/min_length": 1510.0,
      "completions/min_terminated_length": 1510.0,
      "entropy": 0.3890461858827621,
      "epoch": 0.16,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08656251430511475,
      "kl": 0.0022164108740980737,
      "learning_rate": 4.036879377760753e-06,
      "loss": -0.0088,
      "num_tokens": 9326486.0,
      "reward": -0.00443749874830246,
      "reward_std": 0.23447294533252716,
      "rewards/alfworld_rollout_reward_func/mean": -0.00443749874830246,
      "rewards/alfworld_rollout_reward_func/std": 0.3096565306186676,
      "sampling/importance_sampling_ratio/max": 2.934976816177368,
      "sampling/importance_sampling_ratio/mean": 0.5524439811706543,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 51.44619369506836,
      "sampling/sampling_logp_difference/mean": 0.02777031436562538,
      "step": 40,
      "step_time": 226.38629827399564
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6446.0,
      "completions/max_terminated_length": 6446.0,
      "completions/mean_length": 3105.40625,
      "completions/mean_terminated_length": 3105.40625,
      "completions/min_length": 566.0,
      "completions/min_terminated_length": 566.0,
      "entropy": 0.3974974802695215,
      "epoch": 0.164,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06484908610582352,
      "kl": 0.001733089880872285,
      "learning_rate": 3.985172126203416e-06,
      "loss": 0.045,
      "num_tokens": 9443363.0,
      "reward": 0.6158124804496765,
      "reward_std": 0.5850986838340759,
      "rewards/alfworld_rollout_reward_func/mean": 0.6158124804496765,
      "rewards/alfworld_rollout_reward_func/std": 0.5625138282775879,
      "sampling/importance_sampling_ratio/max": 2.7842483520507812,
      "sampling/importance_sampling_ratio/mean": 0.5580112338066101,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.5524964332580566,
      "sampling/sampling_logp_difference/mean": 0.02116631343960762,
      "step": 41,
      "step_time": 183.42022220502986
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10331.0,
      "completions/max_terminated_length": 10331.0,
      "completions/mean_length": 9678.8125,
      "completions/mean_terminated_length": 9678.8125,
      "completions/min_length": 6157.0,
      "completions/min_terminated_length": 6157.0,
      "entropy": 0.25405866745859385,
      "epoch": 0.168,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.053937796503305435,
      "kl": 0.0012423141997714993,
      "learning_rate": 3.932463765572506e-06,
      "loss": 0.2671,
      "num_tokens": 9780125.0,
      "reward": -0.00899999588727951,
      "reward_std": 0.20174865424633026,
      "rewards/alfworld_rollout_reward_func/mean": -0.00899999588727951,
      "rewards/alfworld_rollout_reward_func/std": 0.25330790877342224,
      "sampling/importance_sampling_ratio/max": 2.347205638885498,
      "sampling/importance_sampling_ratio/mean": 0.4977620244026184,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7124943733215332,
      "sampling/sampling_logp_difference/mean": 0.014817986637353897,
      "step": 42,
      "step_time": 419.2987745739956
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6369.0,
      "completions/max_terminated_length": 6369.0,
      "completions/mean_length": 4096.40625,
      "completions/mean_terminated_length": 4096.40625,
      "completions/min_length": 851.0,
      "completions/min_terminated_length": 851.0,
      "entropy": 0.421287601813674,
      "epoch": 0.172,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0667397528886795,
      "kl": 0.0017097990621550707,
      "learning_rate": 3.878789824960677e-06,
      "loss": 0.1548,
      "num_tokens": 9926890.0,
      "reward": 0.40693747997283936,
      "reward_std": 0.563187301158905,
      "rewards/alfworld_rollout_reward_func/mean": 0.40693747997283936,
      "rewards/alfworld_rollout_reward_func/std": 0.5423861145973206,
      "sampling/importance_sampling_ratio/max": 1.5852694511413574,
      "sampling/importance_sampling_ratio/mean": 0.2244953215122223,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.694032907485962,
      "sampling/sampling_logp_difference/mean": 0.022100772708654404,
      "step": 43,
      "step_time": 229.39028357601637
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8653.0,
      "completions/max_terminated_length": 8653.0,
      "completions/mean_length": 7266.375,
      "completions/mean_terminated_length": 7266.375,
      "completions/min_length": 2136.0,
      "completions/min_terminated_length": 2136.0,
      "entropy": 0.3246865733526647,
      "epoch": 0.176,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06960894167423248,
      "kl": 0.0016282038141071098,
      "learning_rate": 3.8241864843284974e-06,
      "loss": 0.2291,
      "num_tokens": 10182518.0,
      "reward": 0.1416875123977661,
      "reward_std": 0.4538275897502899,
      "rewards/alfworld_rollout_reward_func/mean": 0.1416875123977661,
      "rewards/alfworld_rollout_reward_func/std": 0.4567718803882599,
      "sampling/importance_sampling_ratio/max": 2.9712493419647217,
      "sampling/importance_sampling_ratio/mean": 0.3745124340057373,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.2307512760162354,
      "sampling/sampling_logp_difference/mean": 0.019990643486380577,
      "step": 44,
      "step_time": 327.09745887000463
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8865.0,
      "completions/max_terminated_length": 8865.0,
      "completions/mean_length": 7749.375,
      "completions/mean_terminated_length": 7749.375,
      "completions/min_length": 4545.0,
      "completions/min_terminated_length": 4545.0,
      "entropy": 0.3092833776026964,
      "epoch": 0.18,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09623318165540695,
      "kl": 0.0016021293340600096,
      "learning_rate": 3.7686905501166392e-06,
      "loss": 0.2155,
      "num_tokens": 10453378.0,
      "reward": 0.01706249639391899,
      "reward_std": 0.29643189907073975,
      "rewards/alfworld_rollout_reward_func/mean": 0.01706249639391899,
      "rewards/alfworld_rollout_reward_func/std": 0.32475021481513977,
      "sampling/importance_sampling_ratio/max": 2.940847158432007,
      "sampling/importance_sampling_ratio/mean": 0.5090895295143127,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7243870496749878,
      "sampling/sampling_logp_difference/mean": 0.018908072263002396,
      "step": 45,
      "step_time": 323.6050897610112
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6393.0,
      "completions/max_terminated_length": 6393.0,
      "completions/mean_length": 5821.90625,
      "completions/mean_terminated_length": 5821.90625,
      "completions/min_length": 4712.0,
      "completions/min_terminated_length": 4712.0,
      "entropy": 0.35458078188821673,
      "epoch": 0.184,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.05855393782258034,
      "kl": 0.001659039109654259,
      "learning_rate": 3.712339430435792e-06,
      "loss": -0.0325,
      "num_tokens": 10658431.0,
      "reward": -0.06668749451637268,
      "reward_std": 0.219809427857399,
      "rewards/alfworld_rollout_reward_func/mean": -0.06668749451637268,
      "rewards/alfworld_rollout_reward_func/std": 0.26888951659202576,
      "sampling/importance_sampling_ratio/max": 1.4364463090896606,
      "sampling/importance_sampling_ratio/mean": 0.34551113843917847,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9569941759109497,
      "sampling/sampling_logp_difference/mean": 0.020003242418169975,
      "step": 46,
      "step_time": 244.42248200399627
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9789.0,
      "completions/max_terminated_length": 9789.0,
      "completions/mean_length": 8566.6875,
      "completions/mean_terminated_length": 8566.6875,
      "completions/min_length": 3894.0,
      "completions/min_terminated_length": 3894.0,
      "entropy": 0.3494706563651562,
      "epoch": 0.188,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.11029686778783798,
      "kl": 0.0019244130344304722,
      "learning_rate": 3.6551711098509906e-06,
      "loss": -0.0316,
      "num_tokens": 10957237.0,
      "reward": -0.021687500178813934,
      "reward_std": 0.21682387590408325,
      "rewards/alfworld_rollout_reward_func/mean": -0.021687500178813934,
      "rewards/alfworld_rollout_reward_func/std": 0.27672794461250305,
      "sampling/importance_sampling_ratio/max": 2.3080437183380127,
      "sampling/importance_sampling_ratio/mean": 0.5886873006820679,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.5755746364593506,
      "sampling/sampling_logp_difference/mean": 0.02144911140203476,
      "step": 47,
      "step_time": 368.61484725401533
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5405.0,
      "completions/max_terminated_length": 5405.0,
      "completions/mean_length": 3693.40625,
      "completions/mean_terminated_length": 3693.40625,
      "completions/min_length": 580.0,
      "completions/min_terminated_length": 580.0,
      "entropy": 0.38441212149336934,
      "epoch": 0.192,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1601746827363968,
      "kl": 0.0019249830202170415,
      "learning_rate": 3.59722412377739e-06,
      "loss": 0.4005,
      "num_tokens": 11093058.0,
      "reward": 0.48787498474121094,
      "reward_std": 0.577678918838501,
      "rewards/alfworld_rollout_reward_func/mean": 0.48787498474121094,
      "rewards/alfworld_rollout_reward_func/std": 0.5583091974258423,
      "sampling/importance_sampling_ratio/max": 2.588526725769043,
      "sampling/importance_sampling_ratio/mean": 0.6237692832946777,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0457005500793457,
      "sampling/sampling_logp_difference/mean": 0.02231193520128727,
      "step": 48,
      "step_time": 188.01608121799654
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5346.0,
      "completions/max_terminated_length": 5346.0,
      "completions/mean_length": 1730.15625,
      "completions/mean_terminated_length": 1730.15625,
      "completions/min_length": 502.0,
      "completions/min_terminated_length": 502.0,
      "entropy": 0.3154917797073722,
      "epoch": 0.196,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.04318661987781525,
      "kl": 0.0018497702694730833,
      "learning_rate": 3.5385375325047167e-06,
      "loss": 0.006,
      "num_tokens": 11163079.0,
      "reward": 0.7643749713897705,
      "reward_std": 0.5001854300498962,
      "rewards/alfworld_rollout_reward_func/mean": 0.7643749713897705,
      "rewards/alfworld_rollout_reward_func/std": 0.4970761835575104,
      "sampling/importance_sampling_ratio/max": 1.8017005920410156,
      "sampling/importance_sampling_ratio/mean": 0.6256942749023438,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9354848861694336,
      "sampling/sampling_logp_difference/mean": 0.020013634115457535,
      "step": 49,
      "step_time": 132.72531051501574
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7131.0,
      "completions/max_terminated_length": 7131.0,
      "completions/mean_length": 6687.125,
      "completions/mean_terminated_length": 6687.125,
      "completions/min_length": 6248.0,
      "completions/min_terminated_length": 6248.0,
      "entropy": 0.32370956102386117,
      "epoch": 0.2,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.05834148824214935,
      "kl": 0.001850873190051061,
      "learning_rate": 3.4791508948679263e-06,
      "loss": -0.1192,
      "num_tokens": 11397771.0,
      "reward": -0.07500000298023224,
      "reward_std": 0.03566236048936844,
      "rewards/alfworld_rollout_reward_func/mean": -0.07500000298023224,
      "rewards/alfworld_rollout_reward_func/std": 0.036367300897836685,
      "sampling/importance_sampling_ratio/max": 2.0694658756256104,
      "sampling/importance_sampling_ratio/mean": 0.5067798495292664,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.538026809692383,
      "sampling/sampling_logp_difference/mean": 0.0192611962556839,
      "step": 50,
      "step_time": 270.121436987014
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10148.0,
      "completions/max_terminated_length": 10148.0,
      "completions/mean_length": 7905.28125,
      "completions/mean_terminated_length": 7905.28125,
      "completions/min_length": 5187.0,
      "completions/min_terminated_length": 5187.0,
      "entropy": 0.36072542797774076,
      "epoch": 0.204,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.052209846675395966,
      "kl": 0.0015278563969332026,
      "learning_rate": 3.4191042415818003e-06,
      "loss": 0.019,
      "num_tokens": 11673364.0,
      "reward": 0.00043749623000621796,
      "reward_std": 0.24839580059051514,
      "rewards/alfworld_rollout_reward_func/mean": 0.00043749623000621796,
      "rewards/alfworld_rollout_reward_func/std": 0.32011598348617554,
      "sampling/importance_sampling_ratio/max": 2.9914634227752686,
      "sampling/importance_sampling_ratio/mean": 0.2837412357330322,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.466139316558838,
      "sampling/sampling_logp_difference/mean": 0.02213486284017563,
      "step": 51,
      "step_time": 356.71560497701284
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9651.0,
      "completions/max_terminated_length": 9651.0,
      "completions/mean_length": 8476.09375,
      "completions/mean_terminated_length": 8476.09375,
      "completions/min_length": 3907.0,
      "completions/min_terminated_length": 3907.0,
      "entropy": 0.35094283381477,
      "epoch": 0.208,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.025499675422906876,
      "kl": 0.002995722001287504,
      "learning_rate": 3.358438048257472e-06,
      "loss": 0.0451,
      "num_tokens": 11969111.0,
      "reward": 0.010937499813735485,
      "reward_std": 0.305189311504364,
      "rewards/alfworld_rollout_reward_func/mean": 0.010937499813735485,
      "rewards/alfworld_rollout_reward_func/std": 0.32462701201438904,
      "sampling/importance_sampling_ratio/max": 0.5397596955299377,
      "sampling/importance_sampling_ratio/mean": 0.14611726999282837,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 46.953086853027344,
      "sampling/sampling_logp_difference/mean": 0.026804283261299133,
      "step": 52,
      "step_time": 379.2088962249618
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5280.0,
      "completions/max_terminated_length": 5280.0,
      "completions/mean_length": 4830.59375,
      "completions/mean_terminated_length": 4830.59375,
      "completions/min_length": 3119.0,
      "completions/min_terminated_length": 3119.0,
      "entropy": 0.3361989436671138,
      "epoch": 0.212,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0473778061568737,
      "kl": 0.0014486200398096116,
      "learning_rate": 3.2971932081190474e-06,
      "loss": 0.0833,
      "num_tokens": 12141098.0,
      "reward": 0.11968749761581421,
      "reward_std": 0.43453487753868103,
      "rewards/alfworld_rollout_reward_func/mean": 0.11968749761581421,
      "rewards/alfworld_rollout_reward_func/std": 0.4217166602611542,
      "sampling/importance_sampling_ratio/max": 2.925384759902954,
      "sampling/importance_sampling_ratio/mean": 0.49754995107650757,
      "sampling/importance_sampling_ratio/min": 1.830689319000585e-26,
      "sampling/sampling_logp_difference/max": 51.96953201293945,
      "sampling/sampling_logp_difference/mean": 0.02179238200187683,
      "step": 53,
      "step_time": 226.6324916079975
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4936.0,
      "completions/max_terminated_length": 4936.0,
      "completions/mean_length": 4371.8125,
      "completions/mean_terminated_length": 4371.8125,
      "completions/min_length": 2437.0,
      "completions/min_terminated_length": 2437.0,
      "entropy": 0.31532803853042424,
      "epoch": 0.216,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06350528448820114,
      "kl": 0.0025059625349967973,
      "learning_rate": 3.235411004438741e-06,
      "loss": -0.0317,
      "num_tokens": 12297252.0,
      "reward": -0.03593749552965164,
      "reward_std": 0.16499024629592896,
      "rewards/alfworld_rollout_reward_func/mean": -0.03593749552965164,
      "rewards/alfworld_rollout_reward_func/std": 0.27497121691703796,
      "sampling/importance_sampling_ratio/max": 1.963118314743042,
      "sampling/importance_sampling_ratio/mean": 0.3731878697872162,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.5245189666748047,
      "sampling/sampling_logp_difference/mean": 0.022694915533065796,
      "step": 54,
      "step_time": 197.86540699698526
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9904.0,
      "completions/max_terminated_length": 9904.0,
      "completions/mean_length": 8189.46875,
      "completions/mean_terminated_length": 8189.46875,
      "completions/min_length": 1833.0,
      "completions/min_terminated_length": 1833.0,
      "entropy": 0.2978189126588404,
      "epoch": 0.22,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09359363466501236,
      "kl": 0.003045422376089846,
      "learning_rate": 3.1731330827090865e-06,
      "loss": 0.0623,
      "num_tokens": 12585235.0,
      "reward": 0.16593748331069946,
      "reward_std": 0.5176264643669128,
      "rewards/alfworld_rollout_reward_func/mean": 0.16593748331069946,
      "rewards/alfworld_rollout_reward_func/std": 0.4924832284450531,
      "sampling/importance_sampling_ratio/max": 1.8492510318756104,
      "sampling/importance_sampling_ratio/mean": 0.3705262541770935,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.8525230884552,
      "sampling/sampling_logp_difference/mean": 0.019484862685203552,
      "step": 55,
      "step_time": 350.1612614990154
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5918.0,
      "completions/max_terminated_length": 5918.0,
      "completions/mean_length": 3393.34375,
      "completions/mean_terminated_length": 3393.34375,
      "completions/min_length": 651.0,
      "completions/min_terminated_length": 651.0,
      "entropy": 0.34432175708934665,
      "epoch": 0.224,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09224288165569305,
      "kl": 0.0017622580817260314,
      "learning_rate": 3.1104014225709787e-06,
      "loss": 0.1772,
      "num_tokens": 12711710.0,
      "reward": 0.678937554359436,
      "reward_std": 0.5203538537025452,
      "rewards/alfworld_rollout_reward_func/mean": 0.678937554359436,
      "rewards/alfworld_rollout_reward_func/std": 0.5227290391921997,
      "sampling/importance_sampling_ratio/max": 1.574124813079834,
      "sampling/importance_sampling_ratio/mean": 0.509259819984436,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9354878664016724,
      "sampling/sampling_logp_difference/mean": 0.022206870838999748,
      "step": 56,
      "step_time": 181.0405164859767
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7124.0,
      "completions/max_terminated_length": 7124.0,
      "completions/mean_length": 6700.78125,
      "completions/mean_terminated_length": 6700.78125,
      "completions/min_length": 6243.0,
      "completions/min_terminated_length": 6243.0,
      "entropy": 0.28787622367963195,
      "epoch": 0.228,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.10691696405410767,
      "kl": 0.0021939450498393853,
      "learning_rate": 3.0472583095164875e-06,
      "loss": 0.115,
      "num_tokens": 12947127.0,
      "reward": -0.07062499970197678,
      "reward_std": 0.038271673023700714,
      "rewards/alfworld_rollout_reward_func/mean": -0.07062499970197678,
      "rewards/alfworld_rollout_reward_func/std": 0.03999495878815651,
      "sampling/importance_sampling_ratio/max": 2.679607391357422,
      "sampling/importance_sampling_ratio/mean": 0.6073682904243469,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 55.57805252075195,
      "sampling/sampling_logp_difference/mean": 0.021005703136324883,
      "step": 57,
      "step_time": 288.05928439999116
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10082.0,
      "completions/max_terminated_length": 10082.0,
      "completions/mean_length": 8829.53125,
      "completions/mean_terminated_length": 8829.53125,
      "completions/min_length": 2344.0,
      "completions/min_terminated_length": 2344.0,
      "entropy": 0.3938624085858464,
      "epoch": 0.232,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.024793501943349838,
      "kl": 0.0019174001045030309,
      "learning_rate": 2.9837463063854995e-06,
      "loss": 0.075,
      "num_tokens": 13255528.0,
      "reward": 0.09918749332427979,
      "reward_std": 0.4494594633579254,
      "rewards/alfworld_rollout_reward_func/mean": 0.09918749332427979,
      "rewards/alfworld_rollout_reward_func/std": 0.43808406591415405,
      "sampling/importance_sampling_ratio/max": 1.1230359077453613,
      "sampling/importance_sampling_ratio/mean": 0.19013339281082153,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.3987617492675781,
      "sampling/sampling_logp_difference/mean": 0.023359594866633415,
      "step": 58,
      "step_time": 384.79790244400647
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6767.0,
      "completions/max_terminated_length": 6767.0,
      "completions/mean_length": 6117.25,
      "completions/mean_terminated_length": 6117.25,
      "completions/min_length": 2412.0,
      "completions/min_terminated_length": 2412.0,
      "entropy": 0.31304956832900643,
      "epoch": 0.236,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.15593616664409637,
      "kl": 0.0019624424621724756,
      "learning_rate": 2.919908224675412e-06,
      "loss": 0.0664,
      "num_tokens": 13471920.0,
      "reward": 0.078187495470047,
      "reward_std": 0.4152945280075073,
      "rewards/alfworld_rollout_reward_func/mean": 0.078187495470047,
      "rewards/alfworld_rollout_reward_func/std": 0.4025660753250122,
      "sampling/importance_sampling_ratio/max": 2.6773667335510254,
      "sampling/importance_sampling_ratio/mean": 0.7078142166137695,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.8257670402526855,
      "sampling/sampling_logp_difference/mean": 0.018376916646957397,
      "step": 59,
      "step_time": 243.16926779602363
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10497.0,
      "completions/max_terminated_length": 10497.0,
      "completions/mean_length": 6348.09375,
      "completions/mean_terminated_length": 6348.09375,
      "completions/min_length": 1995.0,
      "completions/min_terminated_length": 1995.0,
      "entropy": 0.33462378149852157,
      "epoch": 0.24,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0756126195192337,
      "kl": 0.0019320993196743075,
      "learning_rate": 2.8557870956832135e-06,
      "loss": 0.069,
      "num_tokens": 13697939.0,
      "reward": 0.3908125162124634,
      "reward_std": 0.5529048442840576,
      "rewards/alfworld_rollout_reward_func/mean": 0.3908125162124634,
      "rewards/alfworld_rollout_reward_func/std": 0.5859829187393188,
      "sampling/importance_sampling_ratio/max": 2.5291330814361572,
      "sampling/importance_sampling_ratio/mean": 0.59832763671875,
      "sampling/importance_sampling_ratio/min": 1.8816867161356073e-36,
      "sampling/sampling_logp_difference/max": 55.401771545410156,
      "sampling/sampling_logp_difference/mean": 0.026156669482588768,
      "step": 60,
      "step_time": 333.30798891498125
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10013.0,
      "completions/max_terminated_length": 10013.0,
      "completions/mean_length": 7095.25,
      "completions/mean_terminated_length": 7095.25,
      "completions/min_length": 3271.0,
      "completions/min_terminated_length": 3271.0,
      "entropy": 0.2983507779426873,
      "epoch": 0.244,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13681145012378693,
      "kl": 0.0013701042553293519,
      "learning_rate": 2.7914261414993983e-06,
      "loss": -0.3559,
      "num_tokens": 13946043.0,
      "reward": 0.2435625046491623,
      "reward_std": 0.5009546279907227,
      "rewards/alfworld_rollout_reward_func/mean": 0.2435625046491623,
      "rewards/alfworld_rollout_reward_func/std": 0.5082091689109802,
      "sampling/importance_sampling_ratio/max": 2.6903092861175537,
      "sampling/importance_sampling_ratio/mean": 0.5245317220687866,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8178590536117554,
      "sampling/sampling_logp_difference/mean": 0.017456701025366783,
      "step": 61,
      "step_time": 348.6192558580078
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6296.0,
      "completions/max_terminated_length": 6296.0,
      "completions/mean_length": 5744.6875,
      "completions/mean_terminated_length": 5744.6875,
      "completions/min_length": 5380.0,
      "completions/min_terminated_length": 5380.0,
      "entropy": 0.35134059516713023,
      "epoch": 0.248,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.053518082946538925,
      "kl": 0.0014115655303612584,
      "learning_rate": 2.726868745873286e-06,
      "loss": 0.162,
      "num_tokens": 14148753.0,
      "reward": -0.05118750035762787,
      "reward_std": 0.11772816628217697,
      "rewards/alfworld_rollout_reward_func/mean": -0.05118750035762787,
      "rewards/alfworld_rollout_reward_func/std": 0.18837153911590576,
      "sampling/importance_sampling_ratio/max": 2.020504951477051,
      "sampling/importance_sampling_ratio/mean": 0.3469912111759186,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.210434913635254,
      "sampling/sampling_logp_difference/mean": 0.018848638981580734,
      "step": 62,
      "step_time": 263.2906391499855
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5156.0,
      "completions/max_terminated_length": 5156.0,
      "completions/mean_length": 3426.0,
      "completions/mean_terminated_length": 3426.0,
      "completions/min_length": 966.0,
      "completions/min_terminated_length": 966.0,
      "entropy": 0.3707121340557933,
      "epoch": 0.252,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.10240434855222702,
      "kl": 0.0017503286617284175,
      "learning_rate": 2.6621584249693577e-06,
      "loss": -0.0694,
      "num_tokens": 14273137.0,
      "reward": 0.20506250858306885,
      "reward_std": 0.4780830144882202,
      "rewards/alfworld_rollout_reward_func/mean": 0.20506250858306885,
      "rewards/alfworld_rollout_reward_func/std": 0.5578250288963318,
      "sampling/importance_sampling_ratio/max": 2.9424898624420166,
      "sampling/importance_sampling_ratio/mean": 0.427452027797699,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0697102546691895,
      "sampling/sampling_logp_difference/mean": 0.02075943537056446,
      "step": 63,
      "step_time": 211.411874217054
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5314.0,
      "completions/max_terminated_length": 5314.0,
      "completions/mean_length": 4876.09375,
      "completions/mean_terminated_length": 4876.09375,
      "completions/min_length": 2897.0,
      "completions/min_terminated_length": 2897.0,
      "entropy": 0.3531786808744073,
      "epoch": 0.256,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1512635201215744,
      "kl": 0.0014349444863910321,
      "learning_rate": 2.5973387980343447e-06,
      "loss": -0.1416,
      "num_tokens": 14446388.0,
      "reward": 0.03425000607967377,
      "reward_std": 0.3334675431251526,
      "rewards/alfworld_rollout_reward_func/mean": 0.03425000607967377,
      "rewards/alfworld_rollout_reward_func/std": 0.3685097396373749,
      "sampling/importance_sampling_ratio/max": 1.736365556716919,
      "sampling/importance_sampling_ratio/mean": 0.5142776966094971,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.6913089752197266,
      "sampling/sampling_logp_difference/mean": 0.01979970932006836,
      "step": 64,
      "step_time": 223.59076329001982
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5580.0,
      "completions/max_terminated_length": 5580.0,
      "completions/mean_length": 3269.65625,
      "completions/mean_terminated_length": 3269.65625,
      "completions/min_length": 1494.0,
      "completions/min_terminated_length": 1494.0,
      "entropy": 0.3332303389906883,
      "epoch": 0.26,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08191720396280289,
      "kl": 0.0014564547764166491,
      "learning_rate": 2.5324535579948274e-06,
      "loss": 0.1715,
      "num_tokens": 14568553.0,
      "reward": 0.6749374866485596,
      "reward_std": 0.5090417861938477,
      "rewards/alfworld_rollout_reward_func/mean": 0.6749374866485596,
      "rewards/alfworld_rollout_reward_func/std": 0.5119609236717224,
      "sampling/importance_sampling_ratio/max": 2.394534111022949,
      "sampling/importance_sampling_ratio/mean": 0.6475673913955688,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.7812056541442871,
      "sampling/sampling_logp_difference/mean": 0.020332667976617813,
      "step": 65,
      "step_time": 180.13867778603162
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10082.0,
      "completions/max_terminated_length": 10082.0,
      "completions/mean_length": 9725.4375,
      "completions/mean_terminated_length": 9725.4375,
      "completions/min_length": 9354.0,
      "completions/min_terminated_length": 9354.0,
      "entropy": 0.23973807599395514,
      "epoch": 0.264,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08751911669969559,
      "kl": 0.0014137740872683935,
      "learning_rate": 2.4675464420051735e-06,
      "loss": 0.0594,
      "num_tokens": 14906711.0,
      "reward": -0.0637499988079071,
      "reward_std": 0.03966202959418297,
      "rewards/alfworld_rollout_reward_func/mean": -0.0637499988079071,
      "rewards/alfworld_rollout_reward_func/std": 0.04069952666759491,
      "sampling/importance_sampling_ratio/max": 1.4892053604125977,
      "sampling/importance_sampling_ratio/mean": 0.4563600420951843,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0478899478912354,
      "sampling/sampling_logp_difference/mean": 0.015552841126918793,
      "step": 66,
      "step_time": 405.4389447569847
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11351.0,
      "completions/max_terminated_length": 11351.0,
      "completions/mean_length": 9758.0,
      "completions/mean_terminated_length": 9758.0,
      "completions/min_length": 6388.0,
      "completions/min_terminated_length": 6388.0,
      "entropy": 0.3808371350169182,
      "epoch": 0.268,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08409757167100906,
      "kl": 0.0021013484038121533,
      "learning_rate": 2.402661201965656e-06,
      "loss": -0.1214,
      "num_tokens": 15246071.0,
      "reward": -0.041312504559755325,
      "reward_std": 0.16533736884593964,
      "rewards/alfworld_rollout_reward_func/mean": -0.041312504559755325,
      "rewards/alfworld_rollout_reward_func/std": 0.2661644518375397,
      "sampling/importance_sampling_ratio/max": 2.918410301208496,
      "sampling/importance_sampling_ratio/mean": 0.46048682928085327,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9574942588806152,
      "sampling/sampling_logp_difference/mean": 0.023561783134937286,
      "step": 67,
      "step_time": 400.60474235798756
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6660.0,
      "completions/max_terminated_length": 6660.0,
      "completions/mean_length": 4721.375,
      "completions/mean_terminated_length": 4721.375,
      "completions/min_length": 2437.0,
      "completions/min_terminated_length": 2437.0,
      "entropy": 0.39944601943716407,
      "epoch": 0.272,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13155992329120636,
      "kl": 0.004245847638230771,
      "learning_rate": 2.3378415750306423e-06,
      "loss": -0.1467,
      "num_tokens": 15413827.0,
      "reward": 0.14081250131130219,
      "reward_std": 0.5212504267692566,
      "rewards/alfworld_rollout_reward_func/mean": 0.14081250131130219,
      "rewards/alfworld_rollout_reward_func/std": 0.49614405632019043,
      "sampling/importance_sampling_ratio/max": 2.665754556655884,
      "sampling/importance_sampling_ratio/mean": 0.3483412265777588,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 54.71510314941406,
      "sampling/sampling_logp_difference/mean": 0.034075140953063965,
      "step": 68,
      "step_time": 239.04357930700644
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6127.0,
      "completions/max_terminated_length": 6127.0,
      "completions/mean_length": 2618.53125,
      "completions/mean_terminated_length": 2618.53125,
      "completions/min_length": 1345.0,
      "completions/min_terminated_length": 1345.0,
      "entropy": 0.2670084983110428,
      "epoch": 0.276,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07557746767997742,
      "kl": 0.0017455605466238922,
      "learning_rate": 2.2731312541267144e-06,
      "loss": 0.136,
      "num_tokens": 15516372.0,
      "reward": 0.8233749866485596,
      "reward_std": 0.4509342312812805,
      "rewards/alfworld_rollout_reward_func/mean": 0.8233749866485596,
      "rewards/alfworld_rollout_reward_func/std": 0.4393092393875122,
      "sampling/importance_sampling_ratio/max": 2.3624253273010254,
      "sampling/importance_sampling_ratio/mean": 0.5589783787727356,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8002318143844604,
      "sampling/sampling_logp_difference/mean": 0.01815294846892357,
      "step": 69,
      "step_time": 161.2419531490159
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7981.0,
      "completions/max_terminated_length": 7981.0,
      "completions/mean_length": 7565.71875,
      "completions/mean_terminated_length": 7565.71875,
      "completions/min_length": 7246.0,
      "completions/min_terminated_length": 7246.0,
      "entropy": 0.3105633989907801,
      "epoch": 0.28,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0817764550447464,
      "kl": 0.001651243514061207,
      "learning_rate": 2.2085738585006026e-06,
      "loss": 0.1247,
      "num_tokens": 15781163.0,
      "reward": -0.07781250029802322,
      "reward_std": 0.03061940334737301,
      "rewards/alfworld_rollout_reward_func/mean": -0.07781250029802322,
      "rewards/alfworld_rollout_reward_func/std": 0.03159566968679428,
      "sampling/importance_sampling_ratio/max": 2.5979273319244385,
      "sampling/importance_sampling_ratio/mean": 0.5695095062255859,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.5120588541030884,
      "sampling/sampling_logp_difference/mean": 0.018368419259786606,
      "step": 70,
      "step_time": 306.831324933999
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10746.0,
      "completions/max_terminated_length": 10746.0,
      "completions/mean_length": 9905.03125,
      "completions/mean_terminated_length": 9905.03125,
      "completions/min_length": 9340.0,
      "completions/min_terminated_length": 9340.0,
      "entropy": 0.3119946555234492,
      "epoch": 0.284,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0689200609922409,
      "kl": 0.0015134483564906986,
      "learning_rate": 2.1442129043167877e-06,
      "loss": -0.1028,
      "num_tokens": 16125548.0,
      "reward": -0.09656249731779099,
      "reward_std": 0.04267725348472595,
      "rewards/alfworld_rollout_reward_func/mean": -0.09656249731779099,
      "rewards/alfworld_rollout_reward_func/std": 0.04396548494696617,
      "sampling/importance_sampling_ratio/max": 2.8877551555633545,
      "sampling/importance_sampling_ratio/mean": 0.5244022607803345,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.141198754310608,
      "sampling/sampling_logp_difference/mean": 0.01733534410595894,
      "step": 71,
      "step_time": 434.37145673600025
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5787.0,
      "completions/max_terminated_length": 5787.0,
      "completions/mean_length": 5407.4375,
      "completions/mean_terminated_length": 5407.4375,
      "completions/min_length": 4981.0,
      "completions/min_terminated_length": 4981.0,
      "entropy": 0.38375361217185855,
      "epoch": 0.288,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07215612381696701,
      "kl": 0.0014983757901063655,
      "learning_rate": 2.080091775324588e-06,
      "loss": 0.1877,
      "num_tokens": 16316858.0,
      "reward": -0.05406250059604645,
      "reward_std": 0.11169975996017456,
      "rewards/alfworld_rollout_reward_func/mean": -0.05406250059604645,
      "rewards/alfworld_rollout_reward_func/std": 0.18347644805908203,
      "sampling/importance_sampling_ratio/max": 1.1718748807907104,
      "sampling/importance_sampling_ratio/mean": 0.2574652433395386,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8072030544281006,
      "sampling/sampling_logp_difference/mean": 0.020505312830209732,
      "step": 72,
      "step_time": 245.37548473998322
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9718.0,
      "completions/max_terminated_length": 9718.0,
      "completions/mean_length": 7712.53125,
      "completions/mean_terminated_length": 7712.53125,
      "completions/min_length": 3947.0,
      "completions/min_terminated_length": 3947.0,
      "entropy": 0.2971543218009174,
      "epoch": 0.292,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.14229293167591095,
      "kl": 0.0018976893989020027,
      "learning_rate": 2.016253693614501e-06,
      "loss": 0.0903,
      "num_tokens": 16586955.0,
      "reward": 0.07743750512599945,
      "reward_std": 0.4277241826057434,
      "rewards/alfworld_rollout_reward_func/mean": 0.07743750512599945,
      "rewards/alfworld_rollout_reward_func/std": 0.413460373878479,
      "sampling/importance_sampling_ratio/max": 2.993452548980713,
      "sampling/importance_sampling_ratio/mean": 0.56195068359375,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.3662099838256836,
      "sampling/sampling_logp_difference/mean": 0.01890537701547146,
      "step": 73,
      "step_time": 339.4415916220023
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8250.0,
      "completions/max_terminated_length": 8250.0,
      "completions/mean_length": 5758.4375,
      "completions/mean_terminated_length": 5758.4375,
      "completions/min_length": 1738.0,
      "completions/min_terminated_length": 1738.0,
      "entropy": 0.30767737748101354,
      "epoch": 0.296,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0943857803940773,
      "kl": 0.0015604560594510986,
      "learning_rate": 1.9527416904835133e-06,
      "loss": 0.2749,
      "num_tokens": 16794329.0,
      "reward": 0.4194375276565552,
      "reward_std": 0.572472870349884,
      "rewards/alfworld_rollout_reward_func/mean": 0.4194375276565552,
      "rewards/alfworld_rollout_reward_func/std": 0.5662068724632263,
      "sampling/importance_sampling_ratio/max": 2.8990275859832764,
      "sampling/importance_sampling_ratio/mean": 0.6088296175003052,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.829563021659851,
      "sampling/sampling_logp_difference/mean": 0.019225694239139557,
      "step": 74,
      "step_time": 292.0825683399162
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6582.0,
      "completions/max_terminated_length": 6582.0,
      "completions/mean_length": 3619.1875,
      "completions/mean_terminated_length": 3619.1875,
      "completions/min_length": 864.0,
      "completions/min_terminated_length": 864.0,
      "entropy": 0.3182159150019288,
      "epoch": 0.3,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12715977430343628,
      "kl": 0.001645655636821175,
      "learning_rate": 1.889598577429022e-06,
      "loss": 0.308,
      "num_tokens": 16929567.0,
      "reward": 0.6233749985694885,
      "reward_std": 0.5523929595947266,
      "rewards/alfworld_rollout_reward_func/mean": 0.6233749985694885,
      "rewards/alfworld_rollout_reward_func/std": 0.5413455963134766,
      "sampling/importance_sampling_ratio/max": 2.9408795833587646,
      "sampling/importance_sampling_ratio/mean": 0.8273114562034607,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.3129551410675049,
      "sampling/sampling_logp_difference/mean": 0.01962963677942753,
      "step": 75,
      "step_time": 207.1375826299918
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5181.0,
      "completions/max_terminated_length": 5181.0,
      "completions/mean_length": 4334.53125,
      "completions/mean_terminated_length": 4334.53125,
      "completions/min_length": 2063.0,
      "completions/min_terminated_length": 2063.0,
      "entropy": 0.34562692465260625,
      "epoch": 0.304,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08276179432868958,
      "kl": 0.00154796276910929,
      "learning_rate": 1.8268669172909137e-06,
      "loss": 0.1251,
      "num_tokens": 17084432.0,
      "reward": 0.1054375022649765,
      "reward_std": 0.4694763422012329,
      "rewards/alfworld_rollout_reward_func/mean": 0.1054375022649765,
      "rewards/alfworld_rollout_reward_func/std": 0.4684988260269165,
      "sampling/importance_sampling_ratio/max": 1.8063870668411255,
      "sampling/importance_sampling_ratio/mean": 0.39029014110565186,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9038729667663574,
      "sampling/sampling_logp_difference/mean": 0.018809650093317032,
      "step": 76,
      "step_time": 208.1383357859595
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10452.0,
      "completions/max_terminated_length": 10452.0,
      "completions/mean_length": 8302.25,
      "completions/mean_terminated_length": 8302.25,
      "completions/min_length": 2010.0,
      "completions/min_terminated_length": 2010.0,
      "entropy": 0.2984990398399532,
      "epoch": 0.308,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.11235751956701279,
      "kl": 0.0017886007844936103,
      "learning_rate": 1.7645889955612595e-06,
      "loss": -0.0816,
      "num_tokens": 17374744.0,
      "reward": -0.021937498822808266,
      "reward_std": 0.22131139039993286,
      "rewards/alfworld_rollout_reward_func/mean": -0.021937498822808266,
      "rewards/alfworld_rollout_reward_func/std": 0.28010478615760803,
      "sampling/importance_sampling_ratio/max": 2.364161252975464,
      "sampling/importance_sampling_ratio/mean": 0.4629632234573364,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4188671112060547,
      "sampling/sampling_logp_difference/mean": 0.01893581636250019,
      "step": 77,
      "step_time": 399.66260964999674
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9694.0,
      "completions/max_terminated_length": 9694.0,
      "completions/mean_length": 6271.25,
      "completions/mean_terminated_length": 6271.25,
      "completions/min_length": 1489.0,
      "completions/min_terminated_length": 1489.0,
      "entropy": 0.4018568922765553,
      "epoch": 0.312,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13200783729553223,
      "kl": 0.0022579578471777495,
      "learning_rate": 1.7028067918809539e-06,
      "loss": -0.2553,
      "num_tokens": 17594976.0,
      "reward": 0.2406875193119049,
      "reward_std": 0.5118597745895386,
      "rewards/alfworld_rollout_reward_func/mean": 0.2406875193119049,
      "rewards/alfworld_rollout_reward_func/std": 0.5355828404426575,
      "sampling/importance_sampling_ratio/max": 2.479753017425537,
      "sampling/importance_sampling_ratio/mean": 0.3227984309196472,
      "sampling/importance_sampling_ratio/min": 0.002002666238695383,
      "sampling/sampling_logp_difference/max": 2.8022069931030273,
      "sampling/sampling_logp_difference/mean": 0.023829758167266846,
      "step": 78,
      "step_time": 299.9485474039684
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6824.0,
      "completions/max_terminated_length": 6824.0,
      "completions/mean_length": 4525.46875,
      "completions/mean_terminated_length": 4525.46875,
      "completions/min_length": 1223.0,
      "completions/min_terminated_length": 1223.0,
      "entropy": 0.35179836209863424,
      "epoch": 0.316,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09291504323482513,
      "kl": 0.0013656286282639485,
      "learning_rate": 1.6415619517425296e-06,
      "loss": 0.1543,
      "num_tokens": 17756335.0,
      "reward": 0.20887500047683716,
      "reward_std": 0.5421645045280457,
      "rewards/alfworld_rollout_reward_func/mean": 0.20887500047683716,
      "rewards/alfworld_rollout_reward_func/std": 0.5290061235427856,
      "sampling/importance_sampling_ratio/max": 2.4969656467437744,
      "sampling/importance_sampling_ratio/mean": 0.4944171905517578,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.593350887298584,
      "sampling/sampling_logp_difference/mean": 0.018796950578689575,
      "step": 79,
      "step_time": 253.6868438229867
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6889.0,
      "completions/max_terminated_length": 6889.0,
      "completions/mean_length": 5765.34375,
      "completions/mean_terminated_length": 5765.34375,
      "completions/min_length": 1729.0,
      "completions/min_terminated_length": 1729.0,
      "entropy": 0.29748341953381896,
      "epoch": 0.32,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07730242609977722,
      "kl": 0.0013945158752903808,
      "learning_rate": 1.5808957584181997e-06,
      "loss": 0.1198,
      "num_tokens": 17960122.0,
      "reward": 0.04587499797344208,
      "reward_std": 0.2844948172569275,
      "rewards/alfworld_rollout_reward_func/mean": 0.04587499797344208,
      "rewards/alfworld_rollout_reward_func/std": 0.3961002826690674,
      "sampling/importance_sampling_ratio/max": 1.659905195236206,
      "sampling/importance_sampling_ratio/mean": 0.4384477138519287,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0202646255493164,
      "sampling/sampling_logp_difference/mean": 0.017428694292902946,
      "step": 80,
      "step_time": 264.54027753502305
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5182.0,
      "completions/max_terminated_length": 5182.0,
      "completions/mean_length": 4580.71875,
      "completions/mean_terminated_length": 4580.71875,
      "completions/min_length": 4178.0,
      "completions/min_terminated_length": 4178.0,
      "entropy": 0.40502514131367207,
      "epoch": 0.324,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20536404848098755,
      "kl": 0.0015018332487670705,
      "learning_rate": 1.5208491051320745e-06,
      "loss": 0.3905,
      "num_tokens": 18122737.0,
      "reward": -0.019312500953674316,
      "reward_std": 0.2084033489227295,
      "rewards/alfworld_rollout_reward_func/mean": -0.019312500953674316,
      "rewards/alfworld_rollout_reward_func/std": 0.261531263589859,
      "sampling/importance_sampling_ratio/max": 2.869575262069702,
      "sampling/importance_sampling_ratio/mean": 0.4597967863082886,
      "sampling/importance_sampling_ratio/min": 0.007556523662060499,
      "sampling/sampling_logp_difference/max": 0.9722529649734497,
      "sampling/sampling_logp_difference/mean": 0.020884549245238304,
      "step": 81,
      "step_time": 248.1610625180183
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5441.0,
      "completions/max_terminated_length": 5441.0,
      "completions/mean_length": 4244.53125,
      "completions/mean_terminated_length": 4244.53125,
      "completions/min_length": 774.0,
      "completions/min_terminated_length": 774.0,
      "entropy": 0.36706031998619437,
      "epoch": 0.328,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07920122891664505,
      "kl": 0.0016353318824258167,
      "learning_rate": 1.4614624674952843e-06,
      "loss": 0.0544,
      "num_tokens": 18276066.0,
      "reward": 0.327937513589859,
      "reward_std": 0.5772324204444885,
      "rewards/alfworld_rollout_reward_func/mean": 0.327937513589859,
      "rewards/alfworld_rollout_reward_func/std": 0.5514453649520874,
      "sampling/importance_sampling_ratio/max": 2.0194201469421387,
      "sampling/importance_sampling_ratio/mean": 0.4743329584598541,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8554327487945557,
      "sampling/sampling_logp_difference/mean": 0.02121732383966446,
      "step": 82,
      "step_time": 202.4960144519282
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8424.0,
      "completions/max_terminated_length": 8424.0,
      "completions/mean_length": 1882.84375,
      "completions/mean_terminated_length": 1882.84375,
      "completions/min_length": 867.0,
      "completions/min_terminated_length": 867.0,
      "entropy": 0.2702444209717214,
      "epoch": 0.332,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07358190417289734,
      "kl": 0.0028691932539004483,
      "learning_rate": 1.402775876222611e-06,
      "loss": 0.1489,
      "num_tokens": 18360957.0,
      "reward": 0.9632500410079956,
      "reward_std": 0.13971976935863495,
      "rewards/alfworld_rollout_reward_func/mean": 0.9632500410079956,
      "rewards/alfworld_rollout_reward_func/std": 0.2857166528701782,
      "sampling/importance_sampling_ratio/max": 1.8459749221801758,
      "sampling/importance_sampling_ratio/mean": 0.7306889295578003,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.2139201164245605,
      "sampling/sampling_logp_difference/mean": 0.018962852656841278,
      "step": 83,
      "step_time": 191.13928870702512
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6182.0,
      "completions/max_terminated_length": 6182.0,
      "completions/mean_length": 3669.625,
      "completions/mean_terminated_length": 3669.625,
      "completions/min_length": 837.0,
      "completions/min_terminated_length": 837.0,
      "entropy": 0.3000498446635902,
      "epoch": 0.336,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12246961146593094,
      "kl": 0.0016787616987130605,
      "learning_rate": 1.3448288901490094e-06,
      "loss": 0.3255,
      "num_tokens": 18497233.0,
      "reward": 0.7534999847412109,
      "reward_std": 0.39516681432724,
      "rewards/alfworld_rollout_reward_func/mean": 0.7534999847412109,
      "rewards/alfworld_rollout_reward_func/std": 0.46645042300224304,
      "sampling/importance_sampling_ratio/max": 2.5030677318573,
      "sampling/importance_sampling_ratio/mean": 0.5658935904502869,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.45152747631073,
      "sampling/sampling_logp_difference/mean": 0.020223423838615417,
      "step": 84,
      "step_time": 192.9754896939703
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4404.0,
      "completions/max_terminated_length": 4404.0,
      "completions/mean_length": 3087.5,
      "completions/mean_terminated_length": 3087.5,
      "completions/min_length": 454.0,
      "completions/min_terminated_length": 454.0,
      "entropy": 0.3710099817253649,
      "epoch": 0.34,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.11732742190361023,
      "kl": 0.0023317336108448217,
      "learning_rate": 1.2876605695642086e-06,
      "loss": 0.1281,
      "num_tokens": 18611873.0,
      "reward": 0.4307500123977661,
      "reward_std": 0.5411006808280945,
      "rewards/alfworld_rollout_reward_func/mean": 0.4307500123977661,
      "rewards/alfworld_rollout_reward_func/std": 0.5231669545173645,
      "sampling/importance_sampling_ratio/max": 2.7108399868011475,
      "sampling/importance_sampling_ratio/mean": 0.6843256950378418,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8872215747833252,
      "sampling/sampling_logp_difference/mean": 0.022719303146004677,
      "step": 85,
      "step_time": 162.82808318003663
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10096.0,
      "completions/max_terminated_length": 10096.0,
      "completions/mean_length": 8520.375,
      "completions/mean_terminated_length": 8520.375,
      "completions/min_length": 3267.0,
      "completions/min_terminated_length": 3267.0,
      "entropy": 0.31492479564622045,
      "epoch": 0.344,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.061374712735414505,
      "kl": 0.0017404387290298473,
      "learning_rate": 1.2313094498833612e-06,
      "loss": 0.1858,
      "num_tokens": 18910285.0,
      "reward": 0.15693750977516174,
      "reward_std": 0.5100167989730835,
      "rewards/alfworld_rollout_reward_func/mean": 0.15693750977516174,
      "rewards/alfworld_rollout_reward_func/std": 0.49821487069129944,
      "sampling/importance_sampling_ratio/max": 1.8992809057235718,
      "sampling/importance_sampling_ratio/mean": 0.33807021379470825,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.059936761856079,
      "sampling/sampling_logp_difference/mean": 0.020513102412223816,
      "step": 86,
      "step_time": 375.63238986296346
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9171.0,
      "completions/max_terminated_length": 9171.0,
      "completions/mean_length": 5972.375,
      "completions/mean_terminated_length": 5972.375,
      "completions/min_length": 2094.0,
      "completions/min_terminated_length": 2094.0,
      "entropy": 0.3214904679916799,
      "epoch": 0.348,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0639876201748848,
      "kl": 0.001944402048138727,
      "learning_rate": 1.1758135156715043e-06,
      "loss": 0.2002,
      "num_tokens": 19126009.0,
      "reward": 0.542312502861023,
      "reward_std": 0.5635517239570618,
      "rewards/alfworld_rollout_reward_func/mean": 0.542312502861023,
      "rewards/alfworld_rollout_reward_func/std": 0.5554862022399902,
      "sampling/importance_sampling_ratio/max": 2.4035162925720215,
      "sampling/importance_sampling_ratio/mean": 0.48470091819763184,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.181654691696167,
      "sampling/sampling_logp_difference/mean": 0.019984111189842224,
      "step": 87,
      "step_time": 301.96421380495303
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6850.0,
      "completions/max_terminated_length": 6850.0,
      "completions/mean_length": 6028.90625,
      "completions/mean_terminated_length": 6028.90625,
      "completions/min_length": 5420.0,
      "completions/min_terminated_length": 5420.0,
      "entropy": 0.3696739445440471,
      "epoch": 0.352,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.038705118000507355,
      "kl": 0.0015279942108463729,
      "learning_rate": 1.1212101750393236e-06,
      "loss": 0.0118,
      "num_tokens": 19337782.0,
      "reward": -0.10300000011920929,
      "reward_std": 0.132969468832016,
      "rewards/alfworld_rollout_reward_func/mean": -0.10300000011920929,
      "rewards/alfworld_rollout_reward_func/std": 0.19296397268772125,
      "sampling/importance_sampling_ratio/max": 2.475087881088257,
      "sampling/importance_sampling_ratio/mean": 0.3325284719467163,
      "sampling/importance_sampling_ratio/min": 0.006633421406149864,
      "sampling/sampling_logp_difference/max": 2.3745415210723877,
      "sampling/sampling_logp_difference/mean": 0.01984409987926483,
      "step": 88,
      "step_time": 256.6366900990397
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5329.0,
      "completions/max_terminated_length": 5329.0,
      "completions/mean_length": 3667.71875,
      "completions/mean_terminated_length": 3667.71875,
      "completions/min_length": 1062.0,
      "completions/min_terminated_length": 1062.0,
      "entropy": 0.3431046442128718,
      "epoch": 0.356,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.05565686523914337,
      "kl": 0.002557152631197823,
      "learning_rate": 1.0675362344274953e-06,
      "loss": 0.014,
      "num_tokens": 19471757.0,
      "reward": 0.44062501192092896,
      "reward_std": 0.5888760685920715,
      "rewards/alfworld_rollout_reward_func/mean": 0.44062501192092896,
      "rewards/alfworld_rollout_reward_func/std": 0.5690953731536865,
      "sampling/importance_sampling_ratio/max": 2.401775598526001,
      "sampling/importance_sampling_ratio/mean": 0.42710959911346436,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.14093017578125,
      "sampling/sampling_logp_difference/mean": 0.022225676104426384,
      "step": 89,
      "step_time": 193.9669523180055
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6081.0,
      "completions/max_terminated_length": 6081.0,
      "completions/mean_length": 4530.09375,
      "completions/mean_terminated_length": 4530.09375,
      "completions/min_length": 3277.0,
      "completions/min_terminated_length": 3277.0,
      "entropy": 0.4193602460436523,
      "epoch": 0.36,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.15843002498149872,
      "kl": 0.0020286712642700877,
      "learning_rate": 1.0148278737965845e-06,
      "loss": -0.0643,
      "num_tokens": 19632336.0,
      "reward": 0.12999999523162842,
      "reward_std": 0.4567090570926666,
      "rewards/alfworld_rollout_reward_func/mean": 0.12999999523162842,
      "rewards/alfworld_rollout_reward_func/std": 0.45741432905197144,
      "sampling/importance_sampling_ratio/max": 2.779259204864502,
      "sampling/importance_sampling_ratio/mean": 0.5273926854133606,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.931565523147583,
      "sampling/sampling_logp_difference/mean": 0.02205650880932808,
      "step": 90,
      "step_time": 242.78942537099647
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9883.0,
      "completions/max_terminated_length": 9883.0,
      "completions/mean_length": 6145.40625,
      "completions/mean_terminated_length": 6145.40625,
      "completions/min_length": 2700.0,
      "completions/min_terminated_length": 2700.0,
      "entropy": 0.30875492095947266,
      "epoch": 0.364,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1301383525133133,
      "kl": 0.0016387706618843367,
      "learning_rate": 9.63120622239248e-07,
      "loss": 0.2844,
      "num_tokens": 19851965.0,
      "reward": 0.4439375102519989,
      "reward_std": 0.5609219074249268,
      "rewards/alfworld_rollout_reward_func/mean": 0.4439375102519989,
      "rewards/alfworld_rollout_reward_func/std": 0.5704147815704346,
      "sampling/importance_sampling_ratio/max": 2.949549436569214,
      "sampling/importance_sampling_ratio/mean": 0.6360998153686523,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.2484278678894043,
      "sampling/sampling_logp_difference/mean": 0.018874069675803185,
      "step": 91,
      "step_time": 324.5515782940056
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4739.0,
      "completions/max_terminated_length": 4739.0,
      "completions/mean_length": 3070.40625,
      "completions/mean_terminated_length": 3070.40625,
      "completions/min_length": 691.0,
      "completions/min_terminated_length": 691.0,
      "entropy": 0.4392261947505176,
      "epoch": 0.368,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07320296764373779,
      "kl": 0.002107658132445067,
      "learning_rate": 9.124493340311538e-07,
      "loss": 0.2317,
      "num_tokens": 19964394.0,
      "reward": 0.25699999928474426,
      "reward_std": 0.5001701712608337,
      "rewards/alfworld_rollout_reward_func/mean": 0.25699999928474426,
      "rewards/alfworld_rollout_reward_func/std": 0.5264375805854797,
      "sampling/importance_sampling_ratio/max": 1.6854748725891113,
      "sampling/importance_sampling_ratio/mean": 0.3533988893032074,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7995140552520752,
      "sampling/sampling_logp_difference/mean": 0.025006595999002457,
      "step": 92,
      "step_time": 175.92185580494697
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6947.0,
      "completions/max_terminated_length": 6947.0,
      "completions/mean_length": 3786.75,
      "completions/mean_terminated_length": 3786.75,
      "completions/min_length": 724.0,
      "completions/min_terminated_length": 724.0,
      "entropy": 0.2946209688670933,
      "epoch": 0.372,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08050260692834854,
      "kl": 0.0019706511029653484,
      "learning_rate": 8.628481651367876e-07,
      "loss": 0.1288,
      "num_tokens": 20106562.0,
      "reward": 0.6368750333786011,
      "reward_std": 0.5256940126419067,
      "rewards/alfworld_rollout_reward_func/mean": 0.6368750333786011,
      "rewards/alfworld_rollout_reward_func/std": 0.5401507616043091,
      "sampling/importance_sampling_ratio/max": 2.974215507507324,
      "sampling/importance_sampling_ratio/mean": 0.6188757419586182,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8094772100448608,
      "sampling/sampling_logp_difference/mean": 0.020193612203001976,
      "step": 93,
      "step_time": 206.48572037101258
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5326.0,
      "completions/max_terminated_length": 5326.0,
      "completions/mean_length": 4152.09375,
      "completions/mean_terminated_length": 4152.09375,
      "completions/min_length": 2084.0,
      "completions/min_terminated_length": 2084.0,
      "entropy": 0.4083614726550877,
      "epoch": 0.376,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07430887222290039,
      "kl": 0.0020167900111118797,
      "learning_rate": 8.143505501859553e-07,
      "loss": 0.1415,
      "num_tokens": 20254149.0,
      "reward": -0.057624995708465576,
      "reward_std": 0.22674137353897095,
      "rewards/alfworld_rollout_reward_func/mean": -0.057624995708465576,
      "rewards/alfworld_rollout_reward_func/std": 0.28084638714790344,
      "sampling/importance_sampling_ratio/max": 1.6986135244369507,
      "sampling/importance_sampling_ratio/mean": 0.3223665952682495,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.5820741653442383,
      "sampling/sampling_logp_difference/mean": 0.022008778527379036,
      "step": 94,
      "step_time": 228.1358192209882
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11112.0,
      "completions/max_terminated_length": 11112.0,
      "completions/mean_length": 7901.9375,
      "completions/mean_terminated_length": 7901.9375,
      "completions/min_length": 2256.0,
      "completions/min_terminated_length": 2256.0,
      "entropy": 0.3604189087636769,
      "epoch": 0.38,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.11994580924510956,
      "kl": 0.002103939070366323,
      "learning_rate": 7.669891799365284e-07,
      "loss": 0.2477,
      "num_tokens": 20531587.0,
      "reward": 0.1081249937415123,
      "reward_std": 0.3899664282798767,
      "rewards/alfworld_rollout_reward_func/mean": 0.1081249937415123,
      "rewards/alfworld_rollout_reward_func/std": 0.44592028856277466,
      "sampling/importance_sampling_ratio/max": 2.60292649269104,
      "sampling/importance_sampling_ratio/mean": 0.4245090186595917,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.6102848052978516,
      "sampling/sampling_logp_difference/mean": 0.023151205852627754,
      "step": 95,
      "step_time": 380.974294352025
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 6028.0,
      "completions/max_terminated_length": 6028.0,
      "completions/mean_length": 3872.59375,
      "completions/mean_terminated_length": 3872.59375,
      "completions/min_length": 1222.0,
      "completions/min_terminated_length": 1222.0,
      "entropy": 0.2878755168057978,
      "epoch": 0.384,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09124290198087692,
      "kl": 0.0014382937183654576,
      "learning_rate": 7.207959792385999e-07,
      "loss": 0.4866,
      "num_tokens": 20673142.0,
      "reward": 0.390375018119812,
      "reward_std": 0.6189597845077515,
      "rewards/alfworld_rollout_reward_func/mean": 0.390375018119812,
      "rewards/alfworld_rollout_reward_func/std": 0.5916039347648621,
      "sampling/importance_sampling_ratio/max": 1.520223617553711,
      "sampling/importance_sampling_ratio/mean": 0.46078407764434814,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7394518852233887,
      "sampling/sampling_logp_difference/mean": 0.017645301297307014,
      "step": 96,
      "step_time": 206.5706790599943
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 13788.0,
      "completions/max_terminated_length": 13788.0,
      "completions/mean_length": 10405.34375,
      "completions/mean_terminated_length": 10405.34375,
      "completions/min_length": 1455.0,
      "completions/min_terminated_length": 1455.0,
      "entropy": 0.2904400497209281,
      "epoch": 0.388,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06394726037979126,
      "kl": 0.002259159025925328,
      "learning_rate": 6.75802085514925e-07,
      "loss": 0.1384,
      "num_tokens": 21039201.0,
      "reward": 0.15262500941753387,
      "reward_std": 0.3974687457084656,
      "rewards/alfworld_rollout_reward_func/mean": 0.15262500941753387,
      "rewards/alfworld_rollout_reward_func/std": 0.4503324627876282,
      "sampling/importance_sampling_ratio/max": 1.852544903755188,
      "sampling/importance_sampling_ratio/mean": 0.4211999773979187,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.9718213081359863,
      "sampling/sampling_logp_difference/mean": 0.020324187353253365,
      "step": 97,
      "step_time": 493.7330398180784
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9815.0,
      "completions/max_terminated_length": 9815.0,
      "completions/mean_length": 8616.3125,
      "completions/mean_terminated_length": 8616.3125,
      "completions/min_length": 5394.0,
      "completions/min_terminated_length": 5394.0,
      "entropy": 0.29368510097265244,
      "epoch": 0.392,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.053294822573661804,
      "kl": 0.002172048004467797,
      "learning_rate": 6.320378277721342e-07,
      "loss": 0.0355,
      "num_tokens": 21339499.0,
      "reward": -0.05612500011920929,
      "reward_std": 0.12973152101039886,
      "rewards/alfworld_rollout_reward_func/mean": -0.05612500011920929,
      "rewards/alfworld_rollout_reward_func/std": 0.1938219666481018,
      "sampling/importance_sampling_ratio/max": 2.012138843536377,
      "sampling/importance_sampling_ratio/mean": 0.4920834004878998,
      "sampling/importance_sampling_ratio/min": 5.605193857299268e-45,
      "sampling/sampling_logp_difference/max": 47.1961555480957,
      "sampling/sampling_logp_difference/mean": 0.026191767305135727,
      "step": 98,
      "step_time": 378.82437252601085
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5773.0,
      "completions/max_terminated_length": 5773.0,
      "completions/mean_length": 4738.5,
      "completions/mean_terminated_length": 4738.5,
      "completions/min_length": 1107.0,
      "completions/min_terminated_length": 1107.0,
      "entropy": 0.3817450860515237,
      "epoch": 0.396,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13572344183921814,
      "kl": 0.001779558529960923,
      "learning_rate": 5.895327061568776e-07,
      "loss": 0.1925,
      "num_tokens": 21508315.0,
      "reward": 0.09443750977516174,
      "reward_std": 0.3808988630771637,
      "rewards/alfworld_rollout_reward_func/mean": 0.09443750977516174,
      "rewards/alfworld_rollout_reward_func/std": 0.43921801447868347,
      "sampling/importance_sampling_ratio/max": 2.9193501472473145,
      "sampling/importance_sampling_ratio/mean": 0.6009063720703125,
      "sampling/importance_sampling_ratio/min": 0.01660267822444439,
      "sampling/sampling_logp_difference/max": 1.1129672527313232,
      "sampling/sampling_logp_difference/mean": 0.02005705237388611,
      "step": 99,
      "step_time": 232.97595874097897
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7291.0,
      "completions/max_terminated_length": 7291.0,
      "completions/mean_length": 5083.0625,
      "completions/mean_terminated_length": 5083.0625,
      "completions/min_length": 1140.0,
      "completions/min_terminated_length": 1140.0,
      "entropy": 0.34929203847423196,
      "epoch": 0.4,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13283155858516693,
      "kl": 0.0015651150179110118,
      "learning_rate": 5.483153720706799e-07,
      "loss": -0.0391,
      "num_tokens": 21690781.0,
      "reward": 0.4309374690055847,
      "reward_std": 0.5609171390533447,
      "rewards/alfworld_rollout_reward_func/mean": 0.4309374690055847,
      "rewards/alfworld_rollout_reward_func/std": 0.5701945424079895,
      "sampling/importance_sampling_ratio/max": 2.6785364151000977,
      "sampling/importance_sampling_ratio/mean": 0.6643563508987427,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.5328056812286377,
      "sampling/sampling_logp_difference/mean": 0.020073113963007927,
      "step": 100,
      "step_time": 262.673913760038
    }
  ],
  "logging_steps": 1,
  "max_steps": 125,
  "num_input_tokens_seen": 21690781,
  "num_train_epochs": 1,
  "save_steps": 25,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}