| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.4, |
| "eval_steps": 500, |
| "global_step": 100, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9609.0, |
| "completions/max_terminated_length": 9609.0, |
| "completions/mean_length": 7527.34375, |
| "completions/mean_terminated_length": 7527.34375, |
| "completions/min_length": 2464.0, |
| "completions/min_terminated_length": 2464.0, |
| "entropy": 0.31753343041054904, |
| "epoch": 0.004, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09699171781539917, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": -0.1255, |
| "num_tokens": 263947.0, |
| "reward": 0.051750000566244125, |
| "reward_std": 0.32494160532951355, |
| "rewards/alfworld_rollout_reward_func/mean": 0.051750000566244125, |
| "rewards/alfworld_rollout_reward_func/std": 0.3600812554359436, |
| "sampling/importance_sampling_ratio/max": 1.4741449356079102, |
| "sampling/importance_sampling_ratio/mean": 0.29983627796173096, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.5447624921798706, |
| "sampling/sampling_logp_difference/mean": 0.018553823232650757, |
| "step": 1, |
| "step_time": 334.94067962296685 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9708.0, |
| "completions/max_terminated_length": 9708.0, |
| "completions/mean_length": 7452.15625, |
| "completions/mean_terminated_length": 7452.15625, |
| "completions/min_length": 1958.0, |
| "completions/min_terminated_length": 1958.0, |
| "entropy": 0.26579508977010846, |
| "epoch": 0.008, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07386978715658188, |
| "kl": 0.0, |
| "learning_rate": 1.25e-06, |
| "loss": 0.2055, |
| "num_tokens": 528400.0, |
| "reward": 0.2601250112056732, |
| "reward_std": 0.5141350030899048, |
| "rewards/alfworld_rollout_reward_func/mean": 0.2601250112056732, |
| "rewards/alfworld_rollout_reward_func/std": 0.5191444158554077, |
| "sampling/importance_sampling_ratio/max": 2.4382495880126953, |
| "sampling/importance_sampling_ratio/mean": 0.5181977152824402, |
| "sampling/importance_sampling_ratio/min": 0.00792621448636055, |
| "sampling/sampling_logp_difference/max": 2.3046507835388184, |
| "sampling/sampling_logp_difference/mean": 0.017045794054865837, |
| "step": 2, |
| "step_time": 321.73902505297883 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7379.0, |
| "completions/max_terminated_length": 7379.0, |
| "completions/mean_length": 5838.96875, |
| "completions/mean_terminated_length": 5838.96875, |
| "completions/min_length": 1234.0, |
| "completions/min_terminated_length": 1234.0, |
| "entropy": 0.405664736405015, |
| "epoch": 0.012, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10100319981575012, |
| "kl": 0.0017308223832515068, |
| "learning_rate": 2.5e-06, |
| "loss": -0.079, |
| "num_tokens": 734703.0, |
| "reward": 0.140687495470047, |
| "reward_std": 0.4871714115142822, |
| "rewards/alfworld_rollout_reward_func/mean": 0.140687495470047, |
| "rewards/alfworld_rollout_reward_func/std": 0.47618311643600464, |
| "sampling/importance_sampling_ratio/max": 2.0904979705810547, |
| "sampling/importance_sampling_ratio/mean": 0.32961514592170715, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.584744930267334, |
| "sampling/sampling_logp_difference/mean": 0.022431109100580215, |
| "step": 3, |
| "step_time": 272.60163002300396 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10150.0, |
| "completions/max_terminated_length": 10150.0, |
| "completions/mean_length": 8443.78125, |
| "completions/mean_terminated_length": 8443.78125, |
| "completions/min_length": 5169.0, |
| "completions/min_terminated_length": 5169.0, |
| "entropy": 0.3736097775399685, |
| "epoch": 0.016, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.057365067303180695, |
| "kl": 0.0021683564627892338, |
| "learning_rate": 3.7500000000000005e-06, |
| "loss": -0.1302, |
| "num_tokens": 1027944.0, |
| "reward": -0.03568749502301216, |
| "reward_std": 0.16881901025772095, |
| "rewards/alfworld_rollout_reward_func/mean": -0.03568749502301216, |
| "rewards/alfworld_rollout_reward_func/std": 0.27729666233062744, |
| "sampling/importance_sampling_ratio/max": 0.9850316047668457, |
| "sampling/importance_sampling_ratio/mean": 0.23796725273132324, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.0175373554229736, |
| "sampling/sampling_logp_difference/mean": 0.022764241322875023, |
| "step": 4, |
| "step_time": 360.9181504939843 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5416.0, |
| "completions/max_terminated_length": 5416.0, |
| "completions/mean_length": 4855.84375, |
| "completions/mean_terminated_length": 4855.84375, |
| "completions/min_length": 4281.0, |
| "completions/min_terminated_length": 4281.0, |
| "entropy": 0.3696938706561923, |
| "epoch": 0.02, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.16541753709316254, |
| "kl": 0.001998385167098604, |
| "learning_rate": 5e-06, |
| "loss": -0.3616, |
| "num_tokens": 1199523.0, |
| "reward": -0.09624999761581421, |
| "reward_std": 0.046621114015579224, |
| "rewards/alfworld_rollout_reward_func/mean": -0.09624999761581421, |
| "rewards/alfworld_rollout_reward_func/std": 0.04897332936525345, |
| "sampling/importance_sampling_ratio/max": 2.4984867572784424, |
| "sampling/importance_sampling_ratio/mean": 0.5033700466156006, |
| "sampling/importance_sampling_ratio/min": 0.00037635909393429756, |
| "sampling/sampling_logp_difference/max": 3.5292906761169434, |
| "sampling/sampling_logp_difference/mean": 0.022726019844412804, |
| "step": 5, |
| "step_time": 238.88000224298594 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11870.0, |
| "completions/max_terminated_length": 11870.0, |
| "completions/mean_length": 9758.875, |
| "completions/mean_terminated_length": 9758.875, |
| "completions/min_length": 2533.0, |
| "completions/min_terminated_length": 2533.0, |
| "entropy": 0.4001495079137385, |
| "epoch": 0.024, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06449950486421585, |
| "kl": 0.0019289798929094104, |
| "learning_rate": 4.999157413258782e-06, |
| "loss": 0.0376, |
| "num_tokens": 1539231.0, |
| "reward": 0.030937502160668373, |
| "reward_std": 0.336625874042511, |
| "rewards/alfworld_rollout_reward_func/mean": 0.030937502160668373, |
| "rewards/alfworld_rollout_reward_func/std": 0.37901195883750916, |
| "sampling/importance_sampling_ratio/max": 1.6886974573135376, |
| "sampling/importance_sampling_ratio/mean": 0.3790336847305298, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.1409363746643066, |
| "sampling/sampling_logp_difference/mean": 0.023546157404780388, |
| "step": 6, |
| "step_time": 440.73640958298347 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9966.0, |
| "completions/max_terminated_length": 9966.0, |
| "completions/mean_length": 8191.53125, |
| "completions/mean_terminated_length": 8191.53125, |
| "completions/min_length": 7362.0, |
| "completions/min_terminated_length": 7362.0, |
| "entropy": 0.32905709464102983, |
| "epoch": 0.028, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04034750163555145, |
| "kl": 0.0018940924492198974, |
| "learning_rate": 4.996630220997058e-06, |
| "loss": -0.1075, |
| "num_tokens": 1824368.0, |
| "reward": -0.049187496304512024, |
| "reward_std": 0.12530304491519928, |
| "rewards/alfworld_rollout_reward_func/mean": -0.049187496304512024, |
| "rewards/alfworld_rollout_reward_func/std": 0.1951422095298767, |
| "sampling/importance_sampling_ratio/max": 2.443614959716797, |
| "sampling/importance_sampling_ratio/mean": 0.3964727818965912, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.6266517639160156, |
| "sampling/sampling_logp_difference/mean": 0.020559251308441162, |
| "step": 7, |
| "step_time": 350.02132217200415 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5095.0, |
| "completions/max_terminated_length": 5095.0, |
| "completions/mean_length": 4314.65625, |
| "completions/mean_terminated_length": 4314.65625, |
| "completions/min_length": 736.0, |
| "completions/min_terminated_length": 736.0, |
| "entropy": 0.3601735474076122, |
| "epoch": 0.032, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08598003536462784, |
| "kl": 0.0017114684615080478, |
| "learning_rate": 4.992420126717784e-06, |
| "loss": 0.2628, |
| "num_tokens": 1978501.0, |
| "reward": 0.09481249749660492, |
| "reward_std": 0.3786434829235077, |
| "rewards/alfworld_rollout_reward_func/mean": 0.09481249749660492, |
| "rewards/alfworld_rollout_reward_func/std": 0.430143803358078, |
| "sampling/importance_sampling_ratio/max": 2.776649236679077, |
| "sampling/importance_sampling_ratio/mean": 0.47567594051361084, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.7654576301574707, |
| "sampling/sampling_logp_difference/mean": 0.019868386909365654, |
| "step": 8, |
| "step_time": 225.1138117579976 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9537.0, |
| "completions/max_terminated_length": 9537.0, |
| "completions/mean_length": 7083.96875, |
| "completions/mean_terminated_length": 7083.96875, |
| "completions/min_length": 1562.0, |
| "completions/min_terminated_length": 1562.0, |
| "entropy": 0.30733060790225863, |
| "epoch": 0.036, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09293514490127563, |
| "kl": 0.0028748424542754947, |
| "learning_rate": 4.986529968316654e-06, |
| "loss": 0.2734, |
| "num_tokens": 2231140.0, |
| "reward": 0.3280624747276306, |
| "reward_std": 0.5648518204689026, |
| "rewards/alfworld_rollout_reward_func/mean": 0.3280624747276306, |
| "rewards/alfworld_rollout_reward_func/std": 0.5717496871948242, |
| "sampling/importance_sampling_ratio/max": 2.961937665939331, |
| "sampling/importance_sampling_ratio/mean": 0.5576246380805969, |
| "sampling/importance_sampling_ratio/min": 0.05429670959711075, |
| "sampling/sampling_logp_difference/max": 2.148742198944092, |
| "sampling/sampling_logp_difference/mean": 0.019244220107793808, |
| "step": 9, |
| "step_time": 323.0066561399726 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9496.0, |
| "completions/max_terminated_length": 9496.0, |
| "completions/mean_length": 7961.59375, |
| "completions/mean_terminated_length": 7961.59375, |
| "completions/min_length": 7039.0, |
| "completions/min_terminated_length": 7039.0, |
| "entropy": 0.3252652743831277, |
| "epoch": 0.04, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05244140699505806, |
| "kl": 0.002839431928805425, |
| "learning_rate": 4.978963716169166e-06, |
| "loss": -0.1758, |
| "num_tokens": 2508535.0, |
| "reward": -0.0949999988079071, |
| "reward_std": 0.034666046500205994, |
| "rewards/alfworld_rollout_reward_func/mean": -0.0949999988079071, |
| "rewards/alfworld_rollout_reward_func/std": 0.04079848527908325, |
| "sampling/importance_sampling_ratio/max": 1.3879799842834473, |
| "sampling/importance_sampling_ratio/mean": 0.34057629108428955, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 47.49563217163086, |
| "sampling/sampling_logp_difference/mean": 0.027333300560712814, |
| "step": 10, |
| "step_time": 328.04954918399017 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5477.0, |
| "completions/max_terminated_length": 5477.0, |
| "completions/mean_length": 5025.0, |
| "completions/mean_terminated_length": 5025.0, |
| "completions/min_length": 3290.0, |
| "completions/min_terminated_length": 3290.0, |
| "entropy": 0.37359505658969283, |
| "epoch": 0.044, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07209861278533936, |
| "kl": 0.0015697149883635575, |
| "learning_rate": 4.9697264704543135e-06, |
| "loss": 0.0374, |
| "num_tokens": 2686583.0, |
| "reward": -0.0533750019967556, |
| "reward_std": 0.12613213062286377, |
| "rewards/alfworld_rollout_reward_func/mean": -0.0533750019967556, |
| "rewards/alfworld_rollout_reward_func/std": 0.19394871592521667, |
| "sampling/importance_sampling_ratio/max": 2.769270658493042, |
| "sampling/importance_sampling_ratio/mean": 0.48209959268569946, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 54.623374938964844, |
| "sampling/sampling_logp_difference/mean": 0.02327369526028633, |
| "step": 11, |
| "step_time": 223.6848387320133 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9316.0, |
| "completions/max_terminated_length": 9316.0, |
| "completions/mean_length": 8667.5, |
| "completions/mean_terminated_length": 8667.5, |
| "completions/min_length": 8263.0, |
| "completions/min_terminated_length": 8263.0, |
| "entropy": 0.29008231218904257, |
| "epoch": 0.048, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.036121197044849396, |
| "kl": 0.0016753725431044586, |
| "learning_rate": 4.958824457716707e-06, |
| "loss": -0.0967, |
| "num_tokens": 2988551.0, |
| "reward": -0.08687499910593033, |
| "reward_std": 0.043158818036317825, |
| "rewards/alfworld_rollout_reward_func/mean": -0.08687499910593033, |
| "rewards/alfworld_rollout_reward_func/std": 0.046102095395326614, |
| "sampling/importance_sampling_ratio/max": 1.5300953388214111, |
| "sampling/importance_sampling_ratio/mean": 0.3308807611465454, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7159793376922607, |
| "sampling/sampling_logp_difference/mean": 0.018939001485705376, |
| "step": 12, |
| "step_time": 356.27002431899746 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9878.0, |
| "completions/max_terminated_length": 9878.0, |
| "completions/mean_length": 8581.75, |
| "completions/mean_terminated_length": 8581.75, |
| "completions/min_length": 7974.0, |
| "completions/min_terminated_length": 7974.0, |
| "entropy": 0.2544857235625386, |
| "epoch": 0.052, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.12804464995861053, |
| "kl": 0.0013123316457495093, |
| "learning_rate": 4.9462650266694546e-06, |
| "loss": -0.2577, |
| "num_tokens": 3287647.0, |
| "reward": -0.02643749862909317, |
| "reward_std": 0.12068305909633636, |
| "rewards/alfworld_rollout_reward_func/mean": -0.02643749862909317, |
| "rewards/alfworld_rollout_reward_func/std": 0.18352726101875305, |
| "sampling/importance_sampling_ratio/max": 2.085613965988159, |
| "sampling/importance_sampling_ratio/mean": 0.6230069398880005, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 5.174288749694824, |
| "sampling/sampling_logp_difference/mean": 0.01557723619043827, |
| "step": 13, |
| "step_time": 371.6143698649903 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5113.0, |
| "completions/max_terminated_length": 5113.0, |
| "completions/mean_length": 3425.375, |
| "completions/mean_terminated_length": 3425.375, |
| "completions/min_length": 1258.0, |
| "completions/min_terminated_length": 1258.0, |
| "entropy": 0.3549840818159282, |
| "epoch": 0.056, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06998535245656967, |
| "kl": 0.0019005965677934, |
| "learning_rate": 4.9320566432406185e-06, |
| "loss": 0.0376, |
| "num_tokens": 3412747.0, |
| "reward": 0.4362499713897705, |
| "reward_std": 0.5676650404930115, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4362499713897705, |
| "rewards/alfworld_rollout_reward_func/std": 0.5570400357246399, |
| "sampling/importance_sampling_ratio/max": 2.0355820655822754, |
| "sampling/importance_sampling_ratio/mean": 0.41046884655952454, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.2833807468414307, |
| "sampling/sampling_logp_difference/mean": 0.02138603664934635, |
| "step": 14, |
| "step_time": 175.46163126500323 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9538.0, |
| "completions/max_terminated_length": 9538.0, |
| "completions/mean_length": 8678.4375, |
| "completions/mean_terminated_length": 8678.4375, |
| "completions/min_length": 8210.0, |
| "completions/min_terminated_length": 8210.0, |
| "entropy": 0.2878416678868234, |
| "epoch": 0.06, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03550809249281883, |
| "kl": 0.0012492626774474047, |
| "learning_rate": 4.916208884866593e-06, |
| "loss": 0.0712, |
| "num_tokens": 3714873.0, |
| "reward": -0.07062500715255737, |
| "reward_std": 0.04733540862798691, |
| "rewards/alfworld_rollout_reward_func/mean": -0.07062500715255737, |
| "rewards/alfworld_rollout_reward_func/std": 0.04696859046816826, |
| "sampling/importance_sampling_ratio/max": 1.1348778009414673, |
| "sampling/importance_sampling_ratio/mean": 0.31592193245887756, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.4374537467956543, |
| "sampling/sampling_logp_difference/mean": 0.017727136611938477, |
| "step": 15, |
| "step_time": 364.41606586901617 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5416.0, |
| "completions/max_terminated_length": 5416.0, |
| "completions/mean_length": 1625.96875, |
| "completions/mean_terminated_length": 1625.96875, |
| "completions/min_length": 705.0, |
| "completions/min_terminated_length": 705.0, |
| "entropy": 0.27994309086352587, |
| "epoch": 0.064, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.023568756878376007, |
| "kl": 0.0017632305944061955, |
| "learning_rate": 4.8987324340362445e-06, |
| "loss": -0.0524, |
| "num_tokens": 3782456.0, |
| "reward": 0.8765000104904175, |
| "reward_std": 0.3589959144592285, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8765000104904175, |
| "rewards/alfworld_rollout_reward_func/std": 0.39898157119750977, |
| "sampling/importance_sampling_ratio/max": 1.7719885110855103, |
| "sampling/importance_sampling_ratio/mean": 0.5624613165855408, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.212910532951355, |
| "sampling/sampling_logp_difference/mean": 0.021020911633968353, |
| "step": 16, |
| "step_time": 121.96809069301526 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5848.0, |
| "completions/max_terminated_length": 5848.0, |
| "completions/mean_length": 5169.84375, |
| "completions/mean_terminated_length": 5169.84375, |
| "completions/min_length": 3424.0, |
| "completions/min_terminated_length": 3424.0, |
| "entropy": 0.38696050783619285, |
| "epoch": 0.068, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09861446171998978, |
| "kl": 0.0013451038357743528, |
| "learning_rate": 4.879639071090174e-06, |
| "loss": 0.1571, |
| "num_tokens": 3965683.0, |
| "reward": -0.02056249976158142, |
| "reward_std": 0.22134613990783691, |
| "rewards/alfworld_rollout_reward_func/mean": -0.02056249976158142, |
| "rewards/alfworld_rollout_reward_func/std": 0.2705713212490082, |
| "sampling/importance_sampling_ratio/max": 2.602855682373047, |
| "sampling/importance_sampling_ratio/mean": 0.4220735430717468, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.9502549171447754, |
| "sampling/sampling_logp_difference/mean": 0.02037898264825344, |
| "step": 17, |
| "step_time": 239.7622231810019 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12576.0, |
| "completions/max_terminated_length": 12576.0, |
| "completions/mean_length": 11056.15625, |
| "completions/mean_terminated_length": 11056.15625, |
| "completions/min_length": 4799.0, |
| "completions/min_terminated_length": 4799.0, |
| "entropy": 0.4063271093182266, |
| "epoch": 0.072, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06012088432908058, |
| "kl": 0.0020217235087329755, |
| "learning_rate": 4.858941666279956e-06, |
| "loss": 0.0168, |
| "num_tokens": 4347448.0, |
| "reward": -0.08149999380111694, |
| "reward_std": 0.13932166993618011, |
| "rewards/alfworld_rollout_reward_func/mean": -0.08149999380111694, |
| "rewards/alfworld_rollout_reward_func/std": 0.20591196417808533, |
| "sampling/importance_sampling_ratio/max": 2.379767894744873, |
| "sampling/importance_sampling_ratio/mean": 0.33124375343322754, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7336961030960083, |
| "sampling/sampling_logp_difference/mean": 0.023929107934236526, |
| "step": 18, |
| "step_time": 465.16173633500875 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6323.0, |
| "completions/max_terminated_length": 6323.0, |
| "completions/mean_length": 4971.09375, |
| "completions/mean_terminated_length": 4971.09375, |
| "completions/min_length": 3896.0, |
| "completions/min_terminated_length": 3896.0, |
| "entropy": 0.4407036453485489, |
| "epoch": 0.076, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1755954921245575, |
| "kl": 0.0018943681934615597, |
| "learning_rate": 4.836654171092683e-06, |
| "loss": -0.3381, |
| "num_tokens": 4522491.0, |
| "reward": -0.022624999284744263, |
| "reward_std": 0.20558400452136993, |
| "rewards/alfworld_rollout_reward_func/mean": -0.022624999284744263, |
| "rewards/alfworld_rollout_reward_func/std": 0.26010042428970337, |
| "sampling/importance_sampling_ratio/max": 2.1797361373901367, |
| "sampling/importance_sampling_ratio/mean": 0.3776967525482178, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.9502511024475098, |
| "sampling/sampling_logp_difference/mean": 0.02467840164899826, |
| "step": 19, |
| "step_time": 243.88053144400328 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11207.0, |
| "completions/max_terminated_length": 11207.0, |
| "completions/mean_length": 9984.3125, |
| "completions/mean_terminated_length": 9984.3125, |
| "completions/min_length": 4179.0, |
| "completions/min_terminated_length": 4179.0, |
| "entropy": 0.3350094626657665, |
| "epoch": 0.08, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0771276131272316, |
| "kl": 0.0018198210955233662, |
| "learning_rate": 4.812791608846709e-06, |
| "loss": 0.0206, |
| "num_tokens": 4869989.0, |
| "reward": 0.09581249952316284, |
| "reward_std": 0.44819241762161255, |
| "rewards/alfworld_rollout_reward_func/mean": 0.09581249952316284, |
| "rewards/alfworld_rollout_reward_func/std": 0.43458762764930725, |
| "sampling/importance_sampling_ratio/max": 1.8481526374816895, |
| "sampling/importance_sampling_ratio/mean": 0.38451921939849854, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.429098129272461, |
| "sampling/sampling_logp_difference/mean": 0.020724384114146233, |
| "step": 20, |
| "step_time": 405.4802599009927 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5980.0, |
| "completions/max_terminated_length": 5980.0, |
| "completions/mean_length": 2868.5, |
| "completions/mean_terminated_length": 2868.5, |
| "completions/min_length": 623.0, |
| "completions/min_terminated_length": 623.0, |
| "entropy": 0.3131830710917711, |
| "epoch": 0.084, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.044104401022195816, |
| "kl": 0.0020016076996398624, |
| "learning_rate": 4.787370064564884e-06, |
| "loss": -0.0577, |
| "num_tokens": 4979029.0, |
| "reward": 0.6869999766349792, |
| "reward_std": 0.5118247866630554, |
| "rewards/alfworld_rollout_reward_func/mean": 0.6869999766349792, |
| "rewards/alfworld_rollout_reward_func/std": 0.5184879302978516, |
| "sampling/importance_sampling_ratio/max": 2.4770655632019043, |
| "sampling/importance_sampling_ratio/mean": 0.6952197551727295, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.0350642204284668, |
| "sampling/sampling_logp_difference/mean": 0.020318225026130676, |
| "step": 21, |
| "step_time": 161.65337034598633 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8776.0, |
| "completions/max_terminated_length": 8776.0, |
| "completions/mean_length": 6767.40625, |
| "completions/mean_terminated_length": 6767.40625, |
| "completions/min_length": 2421.0, |
| "completions/min_terminated_length": 2421.0, |
| "entropy": 0.35168757336214185, |
| "epoch": 0.088, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09959151595830917, |
| "kl": 0.0017227518037543632, |
| "learning_rate": 4.760406674132126e-06, |
| "loss": -0.0616, |
| "num_tokens": 5216002.0, |
| "reward": 0.18024998903274536, |
| "reward_std": 0.5159989595413208, |
| "rewards/alfworld_rollout_reward_func/mean": 0.18024998903274536, |
| "rewards/alfworld_rollout_reward_func/std": 0.5085508227348328, |
| "sampling/importance_sampling_ratio/max": 2.815340518951416, |
| "sampling/importance_sampling_ratio/mean": 0.5774703621864319, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.2189407348632812, |
| "sampling/sampling_logp_difference/mean": 0.02067497931420803, |
| "step": 22, |
| "step_time": 312.3654504569786 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5316.0, |
| "completions/max_terminated_length": 5316.0, |
| "completions/mean_length": 4200.6875, |
| "completions/mean_terminated_length": 4200.6875, |
| "completions/min_length": 526.0, |
| "completions/min_terminated_length": 526.0, |
| "entropy": 0.3291297680698335, |
| "epoch": 0.092, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.12509657442569733, |
| "kl": 0.0014415504001590307, |
| "learning_rate": 4.73191961274466e-06, |
| "loss": 0.0168, |
| "num_tokens": 5367672.0, |
| "reward": 0.250249981880188, |
| "reward_std": 0.49357783794403076, |
| "rewards/alfworld_rollout_reward_func/mean": 0.250249981880188, |
| "rewards/alfworld_rollout_reward_func/std": 0.5176389813423157, |
| "sampling/importance_sampling_ratio/max": 2.6394360065460205, |
| "sampling/importance_sampling_ratio/mean": 0.485737681388855, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.9740701913833618, |
| "sampling/sampling_logp_difference/mean": 0.018988091498613358, |
| "step": 23, |
| "step_time": 205.61723882200022 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5118.0, |
| "completions/max_terminated_length": 5118.0, |
| "completions/mean_length": 3482.15625, |
| "completions/mean_terminated_length": 3482.15625, |
| "completions/min_length": 1306.0, |
| "completions/min_terminated_length": 1306.0, |
| "entropy": 0.3946748194284737, |
| "epoch": 0.096, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08480077236890793, |
| "kl": 0.0012077928076905664, |
| "learning_rate": 4.701928082658661e-06, |
| "loss": 0.1749, |
| "num_tokens": 5495421.0, |
| "reward": 0.5172500014305115, |
| "reward_std": 0.5834309458732605, |
| "rewards/alfworld_rollout_reward_func/mean": 0.5172500014305115, |
| "rewards/alfworld_rollout_reward_func/std": 0.5709651708602905, |
| "sampling/importance_sampling_ratio/max": 2.6909029483795166, |
| "sampling/importance_sampling_ratio/mean": 0.5612046718597412, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.8878040313720703, |
| "sampling/sampling_logp_difference/mean": 0.020074568688869476, |
| "step": 24, |
| "step_time": 182.43979760597722 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8626.0, |
| "completions/max_terminated_length": 8626.0, |
| "completions/mean_length": 7605.125, |
| "completions/mean_terminated_length": 7605.125, |
| "completions/min_length": 4181.0, |
| "completions/min_terminated_length": 4181.0, |
| "entropy": 0.34247989440336823, |
| "epoch": 0.1, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.117367684841156, |
| "kl": 0.0018653218503459357, |
| "learning_rate": 4.67045230024661e-06, |
| "loss": -0.4289, |
| "num_tokens": 5760993.0, |
| "reward": -0.047874994575977325, |
| "reward_std": 0.13079281151294708, |
| "rewards/alfworld_rollout_reward_func/mean": -0.047874994575977325, |
| "rewards/alfworld_rollout_reward_func/std": 0.1973174661397934, |
| "sampling/importance_sampling_ratio/max": 2.7526066303253174, |
| "sampling/importance_sampling_ratio/mean": 0.5172563195228577, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.884810447692871, |
| "sampling/sampling_logp_difference/mean": 0.020854920148849487, |
| "step": 25, |
| "step_time": 318.9381142439961 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11381.0, |
| "completions/max_terminated_length": 11381.0, |
| "completions/mean_length": 9974.5, |
| "completions/mean_terminated_length": 9974.5, |
| "completions/min_length": 9364.0, |
| "completions/min_terminated_length": 9364.0, |
| "entropy": 0.3294394067488611, |
| "epoch": 0.104, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0581807978451252, |
| "kl": 0.0015963109199219616, |
| "learning_rate": 4.6375134823700505e-06, |
| "loss": 0.0943, |
| "num_tokens": 6107313.0, |
| "reward": -0.06718749552965164, |
| "reward_std": 0.10619112849235535, |
| "rewards/alfworld_rollout_reward_func/mean": -0.06718749552965164, |
| "rewards/alfworld_rollout_reward_func/std": 0.17335838079452515, |
| "sampling/importance_sampling_ratio/max": 2.373974084854126, |
| "sampling/importance_sampling_ratio/mean": 0.4386903941631317, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.610280990600586, |
| "sampling/sampling_logp_difference/mean": 0.01913020759820938, |
| "step": 26, |
| "step_time": 414.68458038300014 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6944.0, |
| "completions/max_terminated_length": 6944.0, |
| "completions/mean_length": 6449.375, |
| "completions/mean_terminated_length": 6449.375, |
| "completions/min_length": 5182.0, |
| "completions/min_terminated_length": 5182.0, |
| "entropy": 0.290782836265862, |
| "epoch": 0.108, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1415673792362213, |
| "kl": 0.0014344851497298805, |
| "learning_rate": 4.603133832077953e-06, |
| "loss": 0.3656, |
| "num_tokens": 6334429.0, |
| "reward": 0.0663750022649765, |
| "reward_std": 0.2861044108867645, |
| "rewards/alfworld_rollout_reward_func/mean": 0.0663750022649765, |
| "rewards/alfworld_rollout_reward_func/std": 0.4014340043067932, |
| "sampling/importance_sampling_ratio/max": 2.9577462673187256, |
| "sampling/importance_sampling_ratio/mean": 0.4735347628593445, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.694256067276001, |
| "sampling/sampling_logp_difference/mean": 0.017661554738879204, |
| "step": 27, |
| "step_time": 254.20073604799836 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 4959.0, |
| "completions/max_terminated_length": 4959.0, |
| "completions/mean_length": 2397.875, |
| "completions/mean_terminated_length": 2397.875, |
| "completions/min_length": 519.0, |
| "completions/min_terminated_length": 519.0, |
| "entropy": 0.34110196540132165, |
| "epoch": 0.112, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07966114580631256, |
| "kl": 0.0017357735669065733, |
| "learning_rate": 4.567336523640322e-06, |
| "loss": 0.0816, |
| "num_tokens": 6426137.0, |
| "reward": 0.7531875371932983, |
| "reward_std": 0.42287677526474, |
| "rewards/alfworld_rollout_reward_func/mean": 0.7531875371932983, |
| "rewards/alfworld_rollout_reward_func/std": 0.48645731806755066, |
| "sampling/importance_sampling_ratio/max": 2.248128890991211, |
| "sampling/importance_sampling_ratio/mean": 0.5701067447662354, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.0523573160171509, |
| "sampling/sampling_logp_difference/mean": 0.020480383187532425, |
| "step": 28, |
| "step_time": 151.82388260698644 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9672.0, |
| "completions/max_terminated_length": 9672.0, |
| "completions/mean_length": 8832.34375, |
| "completions/mean_terminated_length": 8832.34375, |
| "completions/min_length": 2991.0, |
| "completions/min_terminated_length": 2991.0, |
| "entropy": 0.32463116594590247, |
| "epoch": 0.116, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.055072441697120667, |
| "kl": 0.0014255273326853057, |
| "learning_rate": 4.530145686927126e-06, |
| "loss": 0.1313, |
| "num_tokens": 6734084.0, |
| "reward": -0.022187499329447746, |
| "reward_std": 0.213197261095047, |
| "rewards/alfworld_rollout_reward_func/mean": -0.022187499329447746, |
| "rewards/alfworld_rollout_reward_func/std": 0.2705041170120239, |
| "sampling/importance_sampling_ratio/max": 2.71278715133667, |
| "sampling/importance_sampling_ratio/mean": 0.5353480577468872, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.212910532951355, |
| "sampling/sampling_logp_difference/mean": 0.019132211804389954, |
| "step": 29, |
| "step_time": 386.17356162999204 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10229.0, |
| "completions/max_terminated_length": 10229.0, |
| "completions/mean_length": 7089.03125, |
| "completions/mean_terminated_length": 7089.03125, |
| "completions/min_length": 2106.0, |
| "completions/min_terminated_length": 2106.0, |
| "entropy": 0.34196448139846325, |
| "epoch": 0.12, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0913311019539833, |
| "kl": 0.0014697671704198, |
| "learning_rate": 4.49158639114309e-06, |
| "loss": 0.1673, |
| "num_tokens": 6986309.0, |
| "reward": 0.3622500002384186, |
| "reward_std": 0.5987363457679749, |
| "rewards/alfworld_rollout_reward_func/mean": 0.3622500002384186, |
| "rewards/alfworld_rollout_reward_func/std": 0.5764731168746948, |
| "sampling/importance_sampling_ratio/max": 2.353553533554077, |
| "sampling/importance_sampling_ratio/mean": 0.5580101013183594, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 55.49094009399414, |
| "sampling/sampling_logp_difference/mean": 0.02466428652405739, |
| "step": 30, |
| "step_time": 361.2400383850036 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7853.0, |
| "completions/max_terminated_length": 7853.0, |
| "completions/mean_length": 6668.875, |
| "completions/mean_terminated_length": 6668.875, |
| "completions/min_length": 1438.0, |
| "completions/min_terminated_length": 1438.0, |
| "entropy": 0.25103138061240315, |
| "epoch": 0.124, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.12625202536582947, |
| "kl": 0.0013389530686254147, |
| "learning_rate": 4.45168462792932e-06, |
| "loss": -0.3054, |
| "num_tokens": 7220673.0, |
| "reward": 0.08543749898672104, |
| "reward_std": 0.33267462253570557, |
| "rewards/alfworld_rollout_reward_func/mean": 0.08543749898672104, |
| "rewards/alfworld_rollout_reward_func/std": 0.3945035934448242, |
| "sampling/importance_sampling_ratio/max": 2.542257308959961, |
| "sampling/importance_sampling_ratio/mean": 0.5212557315826416, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7397236824035645, |
| "sampling/sampling_logp_difference/mean": 0.014140520244836807, |
| "step": 31, |
| "step_time": 298.24992308700166 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13314.0, |
| "completions/max_terminated_length": 13314.0, |
| "completions/mean_length": 10635.21875, |
| "completions/mean_terminated_length": 10635.21875, |
| "completions/min_length": 2254.0, |
| "completions/min_terminated_length": 2254.0, |
| "entropy": 0.327601904515177, |
| "epoch": 0.128, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07485339790582657, |
| "kl": 0.0022537091454069014, |
| "learning_rate": 4.410467293843123e-06, |
| "loss": 0.0829, |
| "num_tokens": 7592744.0, |
| "reward": 0.15349999070167542, |
| "reward_std": 0.4711422920227051, |
| "rewards/alfworld_rollout_reward_func/mean": 0.15349999070167542, |
| "rewards/alfworld_rollout_reward_func/std": 0.4708044230937958, |
| "sampling/importance_sampling_ratio/max": 2.957587480545044, |
| "sampling/importance_sampling_ratio/mean": 0.6460099220275879, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.6615216732025146, |
| "sampling/sampling_logp_difference/mean": 0.0228976272046566, |
| "step": 32, |
| "step_time": 485.91699508898455 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5669.0, |
| "completions/max_terminated_length": 5669.0, |
| "completions/mean_length": 5010.8125, |
| "completions/mean_terminated_length": 5010.8125, |
| "completions/min_length": 4272.0, |
| "completions/min_terminated_length": 4272.0, |
| "entropy": 0.3313480073120445, |
| "epoch": 0.132, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11383697390556335, |
| "kl": 0.002066290511720581, |
| "learning_rate": 4.367962172227866e-06, |
| "loss": 0.2319, |
| "num_tokens": 7770434.0, |
| "reward": -0.043312497437000275, |
| "reward_std": 0.21143738925457, |
| "rewards/alfworld_rollout_reward_func/mean": -0.043312497437000275, |
| "rewards/alfworld_rollout_reward_func/std": 0.2677001953125, |
| "sampling/importance_sampling_ratio/max": 1.8964158296585083, |
| "sampling/importance_sampling_ratio/mean": 0.4280845522880554, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 48.41267395019531, |
| "sampling/sampling_logp_difference/mean": 0.026195591315627098, |
| "step": 33, |
| "step_time": 216.44529388601222 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9870.0, |
| "completions/max_terminated_length": 9870.0, |
| "completions/mean_length": 7890.9375, |
| "completions/mean_terminated_length": 7890.9375, |
| "completions/min_length": 1615.0, |
| "completions/min_terminated_length": 1615.0, |
| "entropy": 0.3045849180780351, |
| "epoch": 0.136, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06688809394836426, |
| "kl": 0.001580554380780086, |
| "learning_rate": 4.324197914485075e-06, |
| "loss": 0.1723, |
| "num_tokens": 8048832.0, |
| "reward": 0.18268749117851257, |
| "reward_std": 0.5357717871665955, |
| "rewards/alfworld_rollout_reward_func/mean": 0.18268749117851257, |
| "rewards/alfworld_rollout_reward_func/std": 0.5283057689666748, |
| "sampling/importance_sampling_ratio/max": 1.4296221733093262, |
| "sampling/importance_sampling_ratio/mean": 0.4630698561668396, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 57.18749237060547, |
| "sampling/sampling_logp_difference/mean": 0.0222089234739542, |
| "step": 34, |
| "step_time": 366.9164286329906 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10091.0, |
| "completions/max_terminated_length": 10091.0, |
| "completions/mean_length": 6937.8125, |
| "completions/mean_terminated_length": 6937.8125, |
| "completions/min_length": 2364.0, |
| "completions/min_terminated_length": 2364.0, |
| "entropy": 0.3189437501132488, |
| "epoch": 0.14, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06047705188393593, |
| "kl": 0.0017184474909299752, |
| "learning_rate": 4.279204020761401e-06, |
| "loss": 0.0459, |
| "num_tokens": 8296602.0, |
| "reward": 0.35374999046325684, |
| "reward_std": 0.5226367712020874, |
| "rewards/alfworld_rollout_reward_func/mean": 0.35374999046325684, |
| "rewards/alfworld_rollout_reward_func/std": 0.5893219113349915, |
| "sampling/importance_sampling_ratio/max": 2.7987327575683594, |
| "sampling/importance_sampling_ratio/mean": 0.5202946662902832, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 56.38393020629883, |
| "sampling/sampling_logp_difference/mean": 0.02321350947022438, |
| "step": 35, |
| "step_time": 352.48793815699173 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6439.0, |
| "completions/max_terminated_length": 6439.0, |
| "completions/mean_length": 5805.8125, |
| "completions/mean_terminated_length": 5805.8125, |
| "completions/min_length": 5410.0, |
| "completions/min_terminated_length": 5410.0, |
| "entropy": 0.31256520189344883, |
| "epoch": 0.144, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06142375245690346, |
| "kl": 0.0011960903293584124, |
| "learning_rate": 4.233010820063473e-06, |
| "loss": -0.0928, |
| "num_tokens": 8501748.0, |
| "reward": -0.07906249910593033, |
| "reward_std": 0.03072059527039528, |
| "rewards/alfworld_rollout_reward_func/mean": -0.07906249910593033, |
| "rewards/alfworld_rollout_reward_func/std": 0.031862542033195496, |
| "sampling/importance_sampling_ratio/max": 1.3057942390441895, |
| "sampling/importance_sampling_ratio/mean": 0.35130757093429565, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.0284595489501953, |
| "sampling/sampling_logp_difference/mean": 0.017173780128359795, |
| "step": 36, |
| "step_time": 262.96477444199263 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10178.0, |
| "completions/max_terminated_length": 10178.0, |
| "completions/mean_length": 8620.375, |
| "completions/mean_terminated_length": 8620.375, |
| "completions/min_length": 2023.0, |
| "completions/min_terminated_length": 2023.0, |
| "entropy": 0.3134400031995028, |
| "epoch": 0.148, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.047842346131801605, |
| "kl": 0.001771117205862538, |
| "learning_rate": 4.185649449814046e-06, |
| "loss": 0.0814, |
| "num_tokens": 8803296.0, |
| "reward": 0.016124999150633812, |
| "reward_std": 0.34558093547821045, |
| "rewards/alfworld_rollout_reward_func/mean": 0.016124999150633812, |
| "rewards/alfworld_rollout_reward_func/std": 0.3839482069015503, |
| "sampling/importance_sampling_ratio/max": 2.2736334800720215, |
| "sampling/importance_sampling_ratio/mean": 0.3046373724937439, |
| "sampling/importance_sampling_ratio/min": 8.543683307914326e-28, |
| "sampling/sampling_logp_difference/max": 54.35212707519531, |
| "sampling/sampling_logp_difference/mean": 0.02278374321758747, |
| "step": 37, |
| "step_time": 377.8420012939969 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5236.0, |
| "completions/max_terminated_length": 5236.0, |
| "completions/mean_length": 4332.15625, |
| "completions/mean_terminated_length": 4332.15625, |
| "completions/min_length": 557.0, |
| "completions/min_terminated_length": 557.0, |
| "entropy": 0.39184045884758234, |
| "epoch": 0.152, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13184601068496704, |
| "kl": 0.0018992484310729196, |
| "learning_rate": 4.137151834863213e-06, |
| "loss": 0.0845, |
| "num_tokens": 8958853.0, |
| "reward": 0.10606250166893005, |
| "reward_std": 0.37507200241088867, |
| "rewards/alfworld_rollout_reward_func/mean": 0.10606250166893005, |
| "rewards/alfworld_rollout_reward_func/std": 0.44086700677871704, |
| "sampling/importance_sampling_ratio/max": 2.8395280838012695, |
| "sampling/importance_sampling_ratio/mean": 0.7063838243484497, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 55.17744445800781, |
| "sampling/sampling_logp_difference/mean": 0.02787482924759388, |
| "step": 38, |
| "step_time": 216.3319418339961 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6895.0, |
| "completions/max_terminated_length": 6895.0, |
| "completions/mean_length": 5100.90625, |
| "completions/mean_terminated_length": 5100.90625, |
| "completions/min_length": 1905.0, |
| "completions/min_terminated_length": 1905.0, |
| "entropy": 0.3214424313046038, |
| "epoch": 0.156, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08944570273160934, |
| "kl": 0.0015830010088393465, |
| "learning_rate": 4.087550665968846e-06, |
| "loss": 0.2, |
| "num_tokens": 9141858.0, |
| "reward": 0.32487499713897705, |
| "reward_std": 0.5901329517364502, |
| "rewards/alfworld_rollout_reward_func/mean": 0.32487499713897705, |
| "rewards/alfworld_rollout_reward_func/std": 0.5753059983253479, |
| "sampling/importance_sampling_ratio/max": 2.313490867614746, |
| "sampling/importance_sampling_ratio/mean": 0.576886773109436, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.0487456321716309, |
| "sampling/sampling_logp_difference/mean": 0.019821226596832275, |
| "step": 39, |
| "step_time": 252.95276514600118 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6532.0, |
| "completions/max_terminated_length": 6532.0, |
| "completions/mean_length": 5208.625, |
| "completions/mean_terminated_length": 5208.625, |
| "completions/min_length": 1510.0, |
| "completions/min_terminated_length": 1510.0, |
| "entropy": 0.3890461858827621, |
| "epoch": 0.16, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08656251430511475, |
| "kl": 0.0022164108740980737, |
| "learning_rate": 4.036879377760753e-06, |
| "loss": -0.0088, |
| "num_tokens": 9326486.0, |
| "reward": -0.00443749874830246, |
| "reward_std": 0.23447294533252716, |
| "rewards/alfworld_rollout_reward_func/mean": -0.00443749874830246, |
| "rewards/alfworld_rollout_reward_func/std": 0.3096565306186676, |
| "sampling/importance_sampling_ratio/max": 2.934976816177368, |
| "sampling/importance_sampling_ratio/mean": 0.5524439811706543, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 51.44619369506836, |
| "sampling/sampling_logp_difference/mean": 0.02777031436562538, |
| "step": 40, |
| "step_time": 226.38629827399564 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6446.0, |
| "completions/max_terminated_length": 6446.0, |
| "completions/mean_length": 3105.40625, |
| "completions/mean_terminated_length": 3105.40625, |
| "completions/min_length": 566.0, |
| "completions/min_terminated_length": 566.0, |
| "entropy": 0.3974974802695215, |
| "epoch": 0.164, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06484908610582352, |
| "kl": 0.001733089880872285, |
| "learning_rate": 3.985172126203416e-06, |
| "loss": 0.045, |
| "num_tokens": 9443363.0, |
| "reward": 0.6158124804496765, |
| "reward_std": 0.5850986838340759, |
| "rewards/alfworld_rollout_reward_func/mean": 0.6158124804496765, |
| "rewards/alfworld_rollout_reward_func/std": 0.5625138282775879, |
| "sampling/importance_sampling_ratio/max": 2.7842483520507812, |
| "sampling/importance_sampling_ratio/mean": 0.5580112338066101, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.5524964332580566, |
| "sampling/sampling_logp_difference/mean": 0.02116631343960762, |
| "step": 41, |
| "step_time": 183.42022220502986 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10331.0, |
| "completions/max_terminated_length": 10331.0, |
| "completions/mean_length": 9678.8125, |
| "completions/mean_terminated_length": 9678.8125, |
| "completions/min_length": 6157.0, |
| "completions/min_terminated_length": 6157.0, |
| "entropy": 0.25405866745859385, |
| "epoch": 0.168, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.053937796503305435, |
| "kl": 0.0012423141997714993, |
| "learning_rate": 3.932463765572506e-06, |
| "loss": 0.2671, |
| "num_tokens": 9780125.0, |
| "reward": -0.00899999588727951, |
| "reward_std": 0.20174865424633026, |
| "rewards/alfworld_rollout_reward_func/mean": -0.00899999588727951, |
| "rewards/alfworld_rollout_reward_func/std": 0.25330790877342224, |
| "sampling/importance_sampling_ratio/max": 2.347205638885498, |
| "sampling/importance_sampling_ratio/mean": 0.4977620244026184, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7124943733215332, |
| "sampling/sampling_logp_difference/mean": 0.014817986637353897, |
| "step": 42, |
| "step_time": 419.2987745739956 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6369.0, |
| "completions/max_terminated_length": 6369.0, |
| "completions/mean_length": 4096.40625, |
| "completions/mean_terminated_length": 4096.40625, |
| "completions/min_length": 851.0, |
| "completions/min_terminated_length": 851.0, |
| "entropy": 0.421287601813674, |
| "epoch": 0.172, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0667397528886795, |
| "kl": 0.0017097990621550707, |
| "learning_rate": 3.878789824960677e-06, |
| "loss": 0.1548, |
| "num_tokens": 9926890.0, |
| "reward": 0.40693747997283936, |
| "reward_std": 0.563187301158905, |
| "rewards/alfworld_rollout_reward_func/mean": 0.40693747997283936, |
| "rewards/alfworld_rollout_reward_func/std": 0.5423861145973206, |
| "sampling/importance_sampling_ratio/max": 1.5852694511413574, |
| "sampling/importance_sampling_ratio/mean": 0.2244953215122223, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.694032907485962, |
| "sampling/sampling_logp_difference/mean": 0.022100772708654404, |
| "step": 43, |
| "step_time": 229.39028357601637 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8653.0, |
| "completions/max_terminated_length": 8653.0, |
| "completions/mean_length": 7266.375, |
| "completions/mean_terminated_length": 7266.375, |
| "completions/min_length": 2136.0, |
| "completions/min_terminated_length": 2136.0, |
| "entropy": 0.3246865733526647, |
| "epoch": 0.176, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06960894167423248, |
| "kl": 0.0016282038141071098, |
| "learning_rate": 3.8241864843284974e-06, |
| "loss": 0.2291, |
| "num_tokens": 10182518.0, |
| "reward": 0.1416875123977661, |
| "reward_std": 0.4538275897502899, |
| "rewards/alfworld_rollout_reward_func/mean": 0.1416875123977661, |
| "rewards/alfworld_rollout_reward_func/std": 0.4567718803882599, |
| "sampling/importance_sampling_ratio/max": 2.9712493419647217, |
| "sampling/importance_sampling_ratio/mean": 0.3745124340057373, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.2307512760162354, |
| "sampling/sampling_logp_difference/mean": 0.019990643486380577, |
| "step": 44, |
| "step_time": 327.09745887000463 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8865.0, |
| "completions/max_terminated_length": 8865.0, |
| "completions/mean_length": 7749.375, |
| "completions/mean_terminated_length": 7749.375, |
| "completions/min_length": 4545.0, |
| "completions/min_terminated_length": 4545.0, |
| "entropy": 0.3092833776026964, |
| "epoch": 0.18, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09623318165540695, |
| "kl": 0.0016021293340600096, |
| "learning_rate": 3.7686905501166392e-06, |
| "loss": 0.2155, |
| "num_tokens": 10453378.0, |
| "reward": 0.01706249639391899, |
| "reward_std": 0.29643189907073975, |
| "rewards/alfworld_rollout_reward_func/mean": 0.01706249639391899, |
| "rewards/alfworld_rollout_reward_func/std": 0.32475021481513977, |
| "sampling/importance_sampling_ratio/max": 2.940847158432007, |
| "sampling/importance_sampling_ratio/mean": 0.5090895295143127, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7243870496749878, |
| "sampling/sampling_logp_difference/mean": 0.018908072263002396, |
| "step": 45, |
| "step_time": 323.6050897610112 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6393.0, |
| "completions/max_terminated_length": 6393.0, |
| "completions/mean_length": 5821.90625, |
| "completions/mean_terminated_length": 5821.90625, |
| "completions/min_length": 4712.0, |
| "completions/min_terminated_length": 4712.0, |
| "entropy": 0.35458078188821673, |
| "epoch": 0.184, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05855393782258034, |
| "kl": 0.001659039109654259, |
| "learning_rate": 3.712339430435792e-06, |
| "loss": -0.0325, |
| "num_tokens": 10658431.0, |
| "reward": -0.06668749451637268, |
| "reward_std": 0.219809427857399, |
| "rewards/alfworld_rollout_reward_func/mean": -0.06668749451637268, |
| "rewards/alfworld_rollout_reward_func/std": 0.26888951659202576, |
| "sampling/importance_sampling_ratio/max": 1.4364463090896606, |
| "sampling/importance_sampling_ratio/mean": 0.34551113843917847, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.9569941759109497, |
| "sampling/sampling_logp_difference/mean": 0.020003242418169975, |
| "step": 46, |
| "step_time": 244.42248200399627 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9789.0, |
| "completions/max_terminated_length": 9789.0, |
| "completions/mean_length": 8566.6875, |
| "completions/mean_terminated_length": 8566.6875, |
| "completions/min_length": 3894.0, |
| "completions/min_terminated_length": 3894.0, |
| "entropy": 0.3494706563651562, |
| "epoch": 0.188, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11029686778783798, |
| "kl": 0.0019244130344304722, |
| "learning_rate": 3.6551711098509906e-06, |
| "loss": -0.0316, |
| "num_tokens": 10957237.0, |
| "reward": -0.021687500178813934, |
| "reward_std": 0.21682387590408325, |
| "rewards/alfworld_rollout_reward_func/mean": -0.021687500178813934, |
| "rewards/alfworld_rollout_reward_func/std": 0.27672794461250305, |
| "sampling/importance_sampling_ratio/max": 2.3080437183380127, |
| "sampling/importance_sampling_ratio/mean": 0.5886873006820679, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.5755746364593506, |
| "sampling/sampling_logp_difference/mean": 0.02144911140203476, |
| "step": 47, |
| "step_time": 368.61484725401533 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5405.0, |
| "completions/max_terminated_length": 5405.0, |
| "completions/mean_length": 3693.40625, |
| "completions/mean_terminated_length": 3693.40625, |
| "completions/min_length": 580.0, |
| "completions/min_terminated_length": 580.0, |
| "entropy": 0.38441212149336934, |
| "epoch": 0.192, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1601746827363968, |
| "kl": 0.0019249830202170415, |
| "learning_rate": 3.59722412377739e-06, |
| "loss": 0.4005, |
| "num_tokens": 11093058.0, |
| "reward": 0.48787498474121094, |
| "reward_std": 0.577678918838501, |
| "rewards/alfworld_rollout_reward_func/mean": 0.48787498474121094, |
| "rewards/alfworld_rollout_reward_func/std": 0.5583091974258423, |
| "sampling/importance_sampling_ratio/max": 2.588526725769043, |
| "sampling/importance_sampling_ratio/mean": 0.6237692832946777, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.0457005500793457, |
| "sampling/sampling_logp_difference/mean": 0.02231193520128727, |
| "step": 48, |
| "step_time": 188.01608121799654 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5346.0, |
| "completions/max_terminated_length": 5346.0, |
| "completions/mean_length": 1730.15625, |
| "completions/mean_terminated_length": 1730.15625, |
| "completions/min_length": 502.0, |
| "completions/min_terminated_length": 502.0, |
| "entropy": 0.3154917797073722, |
| "epoch": 0.196, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04318661987781525, |
| "kl": 0.0018497702694730833, |
| "learning_rate": 3.5385375325047167e-06, |
| "loss": 0.006, |
| "num_tokens": 11163079.0, |
| "reward": 0.7643749713897705, |
| "reward_std": 0.5001854300498962, |
| "rewards/alfworld_rollout_reward_func/mean": 0.7643749713897705, |
| "rewards/alfworld_rollout_reward_func/std": 0.4970761835575104, |
| "sampling/importance_sampling_ratio/max": 1.8017005920410156, |
| "sampling/importance_sampling_ratio/mean": 0.6256942749023438, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.9354848861694336, |
| "sampling/sampling_logp_difference/mean": 0.020013634115457535, |
| "step": 49, |
| "step_time": 132.72531051501574 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7131.0, |
| "completions/max_terminated_length": 7131.0, |
| "completions/mean_length": 6687.125, |
| "completions/mean_terminated_length": 6687.125, |
| "completions/min_length": 6248.0, |
| "completions/min_terminated_length": 6248.0, |
| "entropy": 0.32370956102386117, |
| "epoch": 0.2, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05834148824214935, |
| "kl": 0.001850873190051061, |
| "learning_rate": 3.4791508948679263e-06, |
| "loss": -0.1192, |
| "num_tokens": 11397771.0, |
| "reward": -0.07500000298023224, |
| "reward_std": 0.03566236048936844, |
| "rewards/alfworld_rollout_reward_func/mean": -0.07500000298023224, |
| "rewards/alfworld_rollout_reward_func/std": 0.036367300897836685, |
| "sampling/importance_sampling_ratio/max": 2.0694658756256104, |
| "sampling/importance_sampling_ratio/mean": 0.5067798495292664, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.538026809692383, |
| "sampling/sampling_logp_difference/mean": 0.0192611962556839, |
| "step": 50, |
| "step_time": 270.121436987014 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10148.0, |
| "completions/max_terminated_length": 10148.0, |
| "completions/mean_length": 7905.28125, |
| "completions/mean_terminated_length": 7905.28125, |
| "completions/min_length": 5187.0, |
| "completions/min_terminated_length": 5187.0, |
| "entropy": 0.36072542797774076, |
| "epoch": 0.204, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.052209846675395966, |
| "kl": 0.0015278563969332026, |
| "learning_rate": 3.4191042415818003e-06, |
| "loss": 0.019, |
| "num_tokens": 11673364.0, |
| "reward": 0.00043749623000621796, |
| "reward_std": 0.24839580059051514, |
| "rewards/alfworld_rollout_reward_func/mean": 0.00043749623000621796, |
| "rewards/alfworld_rollout_reward_func/std": 0.32011598348617554, |
| "sampling/importance_sampling_ratio/max": 2.9914634227752686, |
| "sampling/importance_sampling_ratio/mean": 0.2837412357330322, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.466139316558838, |
| "sampling/sampling_logp_difference/mean": 0.02213486284017563, |
| "step": 51, |
| "step_time": 356.71560497701284 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9651.0, |
| "completions/max_terminated_length": 9651.0, |
| "completions/mean_length": 8476.09375, |
| "completions/mean_terminated_length": 8476.09375, |
| "completions/min_length": 3907.0, |
| "completions/min_terminated_length": 3907.0, |
| "entropy": 0.35094283381477, |
| "epoch": 0.208, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.025499675422906876, |
| "kl": 0.002995722001287504, |
| "learning_rate": 3.358438048257472e-06, |
| "loss": 0.0451, |
| "num_tokens": 11969111.0, |
| "reward": 0.010937499813735485, |
| "reward_std": 0.305189311504364, |
| "rewards/alfworld_rollout_reward_func/mean": 0.010937499813735485, |
| "rewards/alfworld_rollout_reward_func/std": 0.32462701201438904, |
| "sampling/importance_sampling_ratio/max": 0.5397596955299377, |
| "sampling/importance_sampling_ratio/mean": 0.14611726999282837, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 46.953086853027344, |
| "sampling/sampling_logp_difference/mean": 0.026804283261299133, |
| "step": 52, |
| "step_time": 379.2088962249618 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5280.0, |
| "completions/max_terminated_length": 5280.0, |
| "completions/mean_length": 4830.59375, |
| "completions/mean_terminated_length": 4830.59375, |
| "completions/min_length": 3119.0, |
| "completions/min_terminated_length": 3119.0, |
| "entropy": 0.3361989436671138, |
| "epoch": 0.212, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0473778061568737, |
| "kl": 0.0014486200398096116, |
| "learning_rate": 3.2971932081190474e-06, |
| "loss": 0.0833, |
| "num_tokens": 12141098.0, |
| "reward": 0.11968749761581421, |
| "reward_std": 0.43453487753868103, |
| "rewards/alfworld_rollout_reward_func/mean": 0.11968749761581421, |
| "rewards/alfworld_rollout_reward_func/std": 0.4217166602611542, |
| "sampling/importance_sampling_ratio/max": 2.925384759902954, |
| "sampling/importance_sampling_ratio/mean": 0.49754995107650757, |
| "sampling/importance_sampling_ratio/min": 1.830689319000585e-26, |
| "sampling/sampling_logp_difference/max": 51.96953201293945, |
| "sampling/sampling_logp_difference/mean": 0.02179238200187683, |
| "step": 53, |
| "step_time": 226.6324916079975 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 4936.0, |
| "completions/max_terminated_length": 4936.0, |
| "completions/mean_length": 4371.8125, |
| "completions/mean_terminated_length": 4371.8125, |
| "completions/min_length": 2437.0, |
| "completions/min_terminated_length": 2437.0, |
| "entropy": 0.31532803853042424, |
| "epoch": 0.216, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06350528448820114, |
| "kl": 0.0025059625349967973, |
| "learning_rate": 3.235411004438741e-06, |
| "loss": -0.0317, |
| "num_tokens": 12297252.0, |
| "reward": -0.03593749552965164, |
| "reward_std": 0.16499024629592896, |
| "rewards/alfworld_rollout_reward_func/mean": -0.03593749552965164, |
| "rewards/alfworld_rollout_reward_func/std": 0.27497121691703796, |
| "sampling/importance_sampling_ratio/max": 1.963118314743042, |
| "sampling/importance_sampling_ratio/mean": 0.3731878697872162, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.5245189666748047, |
| "sampling/sampling_logp_difference/mean": 0.022694915533065796, |
| "step": 54, |
| "step_time": 197.86540699698526 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9904.0, |
| "completions/max_terminated_length": 9904.0, |
| "completions/mean_length": 8189.46875, |
| "completions/mean_terminated_length": 8189.46875, |
| "completions/min_length": 1833.0, |
| "completions/min_terminated_length": 1833.0, |
| "entropy": 0.2978189126588404, |
| "epoch": 0.22, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09359363466501236, |
| "kl": 0.003045422376089846, |
| "learning_rate": 3.1731330827090865e-06, |
| "loss": 0.0623, |
| "num_tokens": 12585235.0, |
| "reward": 0.16593748331069946, |
| "reward_std": 0.5176264643669128, |
| "rewards/alfworld_rollout_reward_func/mean": 0.16593748331069946, |
| "rewards/alfworld_rollout_reward_func/std": 0.4924832284450531, |
| "sampling/importance_sampling_ratio/max": 1.8492510318756104, |
| "sampling/importance_sampling_ratio/mean": 0.3705262541770935, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.8525230884552, |
| "sampling/sampling_logp_difference/mean": 0.019484862685203552, |
| "step": 55, |
| "step_time": 350.1612614990154 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5918.0, |
| "completions/max_terminated_length": 5918.0, |
| "completions/mean_length": 3393.34375, |
| "completions/mean_terminated_length": 3393.34375, |
| "completions/min_length": 651.0, |
| "completions/min_terminated_length": 651.0, |
| "entropy": 0.34432175708934665, |
| "epoch": 0.224, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09224288165569305, |
| "kl": 0.0017622580817260314, |
| "learning_rate": 3.1104014225709787e-06, |
| "loss": 0.1772, |
| "num_tokens": 12711710.0, |
| "reward": 0.678937554359436, |
| "reward_std": 0.5203538537025452, |
| "rewards/alfworld_rollout_reward_func/mean": 0.678937554359436, |
| "rewards/alfworld_rollout_reward_func/std": 0.5227290391921997, |
| "sampling/importance_sampling_ratio/max": 1.574124813079834, |
| "sampling/importance_sampling_ratio/mean": 0.509259819984436, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.9354878664016724, |
| "sampling/sampling_logp_difference/mean": 0.022206870838999748, |
| "step": 56, |
| "step_time": 181.0405164859767 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7124.0, |
| "completions/max_terminated_length": 7124.0, |
| "completions/mean_length": 6700.78125, |
| "completions/mean_terminated_length": 6700.78125, |
| "completions/min_length": 6243.0, |
| "completions/min_terminated_length": 6243.0, |
| "entropy": 0.28787622367963195, |
| "epoch": 0.228, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10691696405410767, |
| "kl": 0.0021939450498393853, |
| "learning_rate": 3.0472583095164875e-06, |
| "loss": 0.115, |
| "num_tokens": 12947127.0, |
| "reward": -0.07062499970197678, |
| "reward_std": 0.038271673023700714, |
| "rewards/alfworld_rollout_reward_func/mean": -0.07062499970197678, |
| "rewards/alfworld_rollout_reward_func/std": 0.03999495878815651, |
| "sampling/importance_sampling_ratio/max": 2.679607391357422, |
| "sampling/importance_sampling_ratio/mean": 0.6073682904243469, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 55.57805252075195, |
| "sampling/sampling_logp_difference/mean": 0.021005703136324883, |
| "step": 57, |
| "step_time": 288.05928439999116 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10082.0, |
| "completions/max_terminated_length": 10082.0, |
| "completions/mean_length": 8829.53125, |
| "completions/mean_terminated_length": 8829.53125, |
| "completions/min_length": 2344.0, |
| "completions/min_terminated_length": 2344.0, |
| "entropy": 0.3938624085858464, |
| "epoch": 0.232, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.024793501943349838, |
| "kl": 0.0019174001045030309, |
| "learning_rate": 2.9837463063854995e-06, |
| "loss": 0.075, |
| "num_tokens": 13255528.0, |
| "reward": 0.09918749332427979, |
| "reward_std": 0.4494594633579254, |
| "rewards/alfworld_rollout_reward_func/mean": 0.09918749332427979, |
| "rewards/alfworld_rollout_reward_func/std": 0.43808406591415405, |
| "sampling/importance_sampling_ratio/max": 1.1230359077453613, |
| "sampling/importance_sampling_ratio/mean": 0.19013339281082153, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.3987617492675781, |
| "sampling/sampling_logp_difference/mean": 0.023359594866633415, |
| "step": 58, |
| "step_time": 384.79790244400647 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6767.0, |
| "completions/max_terminated_length": 6767.0, |
| "completions/mean_length": 6117.25, |
| "completions/mean_terminated_length": 6117.25, |
| "completions/min_length": 2412.0, |
| "completions/min_terminated_length": 2412.0, |
| "entropy": 0.31304956832900643, |
| "epoch": 0.236, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.15593616664409637, |
| "kl": 0.0019624424621724756, |
| "learning_rate": 2.919908224675412e-06, |
| "loss": 0.0664, |
| "num_tokens": 13471920.0, |
| "reward": 0.078187495470047, |
| "reward_std": 0.4152945280075073, |
| "rewards/alfworld_rollout_reward_func/mean": 0.078187495470047, |
| "rewards/alfworld_rollout_reward_func/std": 0.4025660753250122, |
| "sampling/importance_sampling_ratio/max": 2.6773667335510254, |
| "sampling/importance_sampling_ratio/mean": 0.7078142166137695, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.8257670402526855, |
| "sampling/sampling_logp_difference/mean": 0.018376916646957397, |
| "step": 59, |
| "step_time": 243.16926779602363 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10497.0, |
| "completions/max_terminated_length": 10497.0, |
| "completions/mean_length": 6348.09375, |
| "completions/mean_terminated_length": 6348.09375, |
| "completions/min_length": 1995.0, |
| "completions/min_terminated_length": 1995.0, |
| "entropy": 0.33462378149852157, |
| "epoch": 0.24, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0756126195192337, |
| "kl": 0.0019320993196743075, |
| "learning_rate": 2.8557870956832135e-06, |
| "loss": 0.069, |
| "num_tokens": 13697939.0, |
| "reward": 0.3908125162124634, |
| "reward_std": 0.5529048442840576, |
| "rewards/alfworld_rollout_reward_func/mean": 0.3908125162124634, |
| "rewards/alfworld_rollout_reward_func/std": 0.5859829187393188, |
| "sampling/importance_sampling_ratio/max": 2.5291330814361572, |
| "sampling/importance_sampling_ratio/mean": 0.59832763671875, |
| "sampling/importance_sampling_ratio/min": 1.8816867161356073e-36, |
| "sampling/sampling_logp_difference/max": 55.401771545410156, |
| "sampling/sampling_logp_difference/mean": 0.026156669482588768, |
| "step": 60, |
| "step_time": 333.30798891498125 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10013.0, |
| "completions/max_terminated_length": 10013.0, |
| "completions/mean_length": 7095.25, |
| "completions/mean_terminated_length": 7095.25, |
| "completions/min_length": 3271.0, |
| "completions/min_terminated_length": 3271.0, |
| "entropy": 0.2983507779426873, |
| "epoch": 0.244, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13681145012378693, |
| "kl": 0.0013701042553293519, |
| "learning_rate": 2.7914261414993983e-06, |
| "loss": -0.3559, |
| "num_tokens": 13946043.0, |
| "reward": 0.2435625046491623, |
| "reward_std": 0.5009546279907227, |
| "rewards/alfworld_rollout_reward_func/mean": 0.2435625046491623, |
| "rewards/alfworld_rollout_reward_func/std": 0.5082091689109802, |
| "sampling/importance_sampling_ratio/max": 2.6903092861175537, |
| "sampling/importance_sampling_ratio/mean": 0.5245317220687866, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.8178590536117554, |
| "sampling/sampling_logp_difference/mean": 0.017456701025366783, |
| "step": 61, |
| "step_time": 348.6192558580078 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6296.0, |
| "completions/max_terminated_length": 6296.0, |
| "completions/mean_length": 5744.6875, |
| "completions/mean_terminated_length": 5744.6875, |
| "completions/min_length": 5380.0, |
| "completions/min_terminated_length": 5380.0, |
| "entropy": 0.35134059516713023, |
| "epoch": 0.248, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.053518082946538925, |
| "kl": 0.0014115655303612584, |
| "learning_rate": 2.726868745873286e-06, |
| "loss": 0.162, |
| "num_tokens": 14148753.0, |
| "reward": -0.05118750035762787, |
| "reward_std": 0.11772816628217697, |
| "rewards/alfworld_rollout_reward_func/mean": -0.05118750035762787, |
| "rewards/alfworld_rollout_reward_func/std": 0.18837153911590576, |
| "sampling/importance_sampling_ratio/max": 2.020504951477051, |
| "sampling/importance_sampling_ratio/mean": 0.3469912111759186, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.210434913635254, |
| "sampling/sampling_logp_difference/mean": 0.018848638981580734, |
| "step": 62, |
| "step_time": 263.2906391499855 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5156.0, |
| "completions/max_terminated_length": 5156.0, |
| "completions/mean_length": 3426.0, |
| "completions/mean_terminated_length": 3426.0, |
| "completions/min_length": 966.0, |
| "completions/min_terminated_length": 966.0, |
| "entropy": 0.3707121340557933, |
| "epoch": 0.252, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10240434855222702, |
| "kl": 0.0017503286617284175, |
| "learning_rate": 2.6621584249693577e-06, |
| "loss": -0.0694, |
| "num_tokens": 14273137.0, |
| "reward": 0.20506250858306885, |
| "reward_std": 0.4780830144882202, |
| "rewards/alfworld_rollout_reward_func/mean": 0.20506250858306885, |
| "rewards/alfworld_rollout_reward_func/std": 0.5578250288963318, |
| "sampling/importance_sampling_ratio/max": 2.9424898624420166, |
| "sampling/importance_sampling_ratio/mean": 0.427452027797699, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.0697102546691895, |
| "sampling/sampling_logp_difference/mean": 0.02075943537056446, |
| "step": 63, |
| "step_time": 211.411874217054 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5314.0, |
| "completions/max_terminated_length": 5314.0, |
| "completions/mean_length": 4876.09375, |
| "completions/mean_terminated_length": 4876.09375, |
| "completions/min_length": 2897.0, |
| "completions/min_terminated_length": 2897.0, |
| "entropy": 0.3531786808744073, |
| "epoch": 0.256, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1512635201215744, |
| "kl": 0.0014349444863910321, |
| "learning_rate": 2.5973387980343447e-06, |
| "loss": -0.1416, |
| "num_tokens": 14446388.0, |
| "reward": 0.03425000607967377, |
| "reward_std": 0.3334675431251526, |
| "rewards/alfworld_rollout_reward_func/mean": 0.03425000607967377, |
| "rewards/alfworld_rollout_reward_func/std": 0.3685097396373749, |
| "sampling/importance_sampling_ratio/max": 1.736365556716919, |
| "sampling/importance_sampling_ratio/mean": 0.5142776966094971, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.6913089752197266, |
| "sampling/sampling_logp_difference/mean": 0.01979970932006836, |
| "step": 64, |
| "step_time": 223.59076329001982 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5580.0, |
| "completions/max_terminated_length": 5580.0, |
| "completions/mean_length": 3269.65625, |
| "completions/mean_terminated_length": 3269.65625, |
| "completions/min_length": 1494.0, |
| "completions/min_terminated_length": 1494.0, |
| "entropy": 0.3332303389906883, |
| "epoch": 0.26, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08191720396280289, |
| "kl": 0.0014564547764166491, |
| "learning_rate": 2.5324535579948274e-06, |
| "loss": 0.1715, |
| "num_tokens": 14568553.0, |
| "reward": 0.6749374866485596, |
| "reward_std": 0.5090417861938477, |
| "rewards/alfworld_rollout_reward_func/mean": 0.6749374866485596, |
| "rewards/alfworld_rollout_reward_func/std": 0.5119609236717224, |
| "sampling/importance_sampling_ratio/max": 2.394534111022949, |
| "sampling/importance_sampling_ratio/mean": 0.6475673913955688, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.7812056541442871, |
| "sampling/sampling_logp_difference/mean": 0.020332667976617813, |
| "step": 65, |
| "step_time": 180.13867778603162 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10082.0, |
| "completions/max_terminated_length": 10082.0, |
| "completions/mean_length": 9725.4375, |
| "completions/mean_terminated_length": 9725.4375, |
| "completions/min_length": 9354.0, |
| "completions/min_terminated_length": 9354.0, |
| "entropy": 0.23973807599395514, |
| "epoch": 0.264, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08751911669969559, |
| "kl": 0.0014137740872683935, |
| "learning_rate": 2.4675464420051735e-06, |
| "loss": 0.0594, |
| "num_tokens": 14906711.0, |
| "reward": -0.0637499988079071, |
| "reward_std": 0.03966202959418297, |
| "rewards/alfworld_rollout_reward_func/mean": -0.0637499988079071, |
| "rewards/alfworld_rollout_reward_func/std": 0.04069952666759491, |
| "sampling/importance_sampling_ratio/max": 1.4892053604125977, |
| "sampling/importance_sampling_ratio/mean": 0.4563600420951843, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.0478899478912354, |
| "sampling/sampling_logp_difference/mean": 0.015552841126918793, |
| "step": 66, |
| "step_time": 405.4389447569847 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11351.0, |
| "completions/max_terminated_length": 11351.0, |
| "completions/mean_length": 9758.0, |
| "completions/mean_terminated_length": 9758.0, |
| "completions/min_length": 6388.0, |
| "completions/min_terminated_length": 6388.0, |
| "entropy": 0.3808371350169182, |
| "epoch": 0.268, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08409757167100906, |
| "kl": 0.0021013484038121533, |
| "learning_rate": 2.402661201965656e-06, |
| "loss": -0.1214, |
| "num_tokens": 15246071.0, |
| "reward": -0.041312504559755325, |
| "reward_std": 0.16533736884593964, |
| "rewards/alfworld_rollout_reward_func/mean": -0.041312504559755325, |
| "rewards/alfworld_rollout_reward_func/std": 0.2661644518375397, |
| "sampling/importance_sampling_ratio/max": 2.918410301208496, |
| "sampling/importance_sampling_ratio/mean": 0.46048682928085327, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.9574942588806152, |
| "sampling/sampling_logp_difference/mean": 0.023561783134937286, |
| "step": 67, |
| "step_time": 400.60474235798756 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6660.0, |
| "completions/max_terminated_length": 6660.0, |
| "completions/mean_length": 4721.375, |
| "completions/mean_terminated_length": 4721.375, |
| "completions/min_length": 2437.0, |
| "completions/min_terminated_length": 2437.0, |
| "entropy": 0.39944601943716407, |
| "epoch": 0.272, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13155992329120636, |
| "kl": 0.004245847638230771, |
| "learning_rate": 2.3378415750306423e-06, |
| "loss": -0.1467, |
| "num_tokens": 15413827.0, |
| "reward": 0.14081250131130219, |
| "reward_std": 0.5212504267692566, |
| "rewards/alfworld_rollout_reward_func/mean": 0.14081250131130219, |
| "rewards/alfworld_rollout_reward_func/std": 0.49614405632019043, |
| "sampling/importance_sampling_ratio/max": 2.665754556655884, |
| "sampling/importance_sampling_ratio/mean": 0.3483412265777588, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 54.71510314941406, |
| "sampling/sampling_logp_difference/mean": 0.034075140953063965, |
| "step": 68, |
| "step_time": 239.04357930700644 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6127.0, |
| "completions/max_terminated_length": 6127.0, |
| "completions/mean_length": 2618.53125, |
| "completions/mean_terminated_length": 2618.53125, |
| "completions/min_length": 1345.0, |
| "completions/min_terminated_length": 1345.0, |
| "entropy": 0.2670084983110428, |
| "epoch": 0.276, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07557746767997742, |
| "kl": 0.0017455605466238922, |
| "learning_rate": 2.2731312541267144e-06, |
| "loss": 0.136, |
| "num_tokens": 15516372.0, |
| "reward": 0.8233749866485596, |
| "reward_std": 0.4509342312812805, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8233749866485596, |
| "rewards/alfworld_rollout_reward_func/std": 0.4393092393875122, |
| "sampling/importance_sampling_ratio/max": 2.3624253273010254, |
| "sampling/importance_sampling_ratio/mean": 0.5589783787727356, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.8002318143844604, |
| "sampling/sampling_logp_difference/mean": 0.01815294846892357, |
| "step": 69, |
| "step_time": 161.2419531490159 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7981.0, |
| "completions/max_terminated_length": 7981.0, |
| "completions/mean_length": 7565.71875, |
| "completions/mean_terminated_length": 7565.71875, |
| "completions/min_length": 7246.0, |
| "completions/min_terminated_length": 7246.0, |
| "entropy": 0.3105633989907801, |
| "epoch": 0.28, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0817764550447464, |
| "kl": 0.001651243514061207, |
| "learning_rate": 2.2085738585006026e-06, |
| "loss": 0.1247, |
| "num_tokens": 15781163.0, |
| "reward": -0.07781250029802322, |
| "reward_std": 0.03061940334737301, |
| "rewards/alfworld_rollout_reward_func/mean": -0.07781250029802322, |
| "rewards/alfworld_rollout_reward_func/std": 0.03159566968679428, |
| "sampling/importance_sampling_ratio/max": 2.5979273319244385, |
| "sampling/importance_sampling_ratio/mean": 0.5695095062255859, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.5120588541030884, |
| "sampling/sampling_logp_difference/mean": 0.018368419259786606, |
| "step": 70, |
| "step_time": 306.831324933999 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10746.0, |
| "completions/max_terminated_length": 10746.0, |
| "completions/mean_length": 9905.03125, |
| "completions/mean_terminated_length": 9905.03125, |
| "completions/min_length": 9340.0, |
| "completions/min_terminated_length": 9340.0, |
| "entropy": 0.3119946555234492, |
| "epoch": 0.284, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0689200609922409, |
| "kl": 0.0015134483564906986, |
| "learning_rate": 2.1442129043167877e-06, |
| "loss": -0.1028, |
| "num_tokens": 16125548.0, |
| "reward": -0.09656249731779099, |
| "reward_std": 0.04267725348472595, |
| "rewards/alfworld_rollout_reward_func/mean": -0.09656249731779099, |
| "rewards/alfworld_rollout_reward_func/std": 0.04396548494696617, |
| "sampling/importance_sampling_ratio/max": 2.8877551555633545, |
| "sampling/importance_sampling_ratio/mean": 0.5244022607803345, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.141198754310608, |
| "sampling/sampling_logp_difference/mean": 0.01733534410595894, |
| "step": 71, |
| "step_time": 434.37145673600025 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5787.0, |
| "completions/max_terminated_length": 5787.0, |
| "completions/mean_length": 5407.4375, |
| "completions/mean_terminated_length": 5407.4375, |
| "completions/min_length": 4981.0, |
| "completions/min_terminated_length": 4981.0, |
| "entropy": 0.38375361217185855, |
| "epoch": 0.288, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07215612381696701, |
| "kl": 0.0014983757901063655, |
| "learning_rate": 2.080091775324588e-06, |
| "loss": 0.1877, |
| "num_tokens": 16316858.0, |
| "reward": -0.05406250059604645, |
| "reward_std": 0.11169975996017456, |
| "rewards/alfworld_rollout_reward_func/mean": -0.05406250059604645, |
| "rewards/alfworld_rollout_reward_func/std": 0.18347644805908203, |
| "sampling/importance_sampling_ratio/max": 1.1718748807907104, |
| "sampling/importance_sampling_ratio/mean": 0.2574652433395386, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.8072030544281006, |
| "sampling/sampling_logp_difference/mean": 0.020505312830209732, |
| "step": 72, |
| "step_time": 245.37548473998322 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9718.0, |
| "completions/max_terminated_length": 9718.0, |
| "completions/mean_length": 7712.53125, |
| "completions/mean_terminated_length": 7712.53125, |
| "completions/min_length": 3947.0, |
| "completions/min_terminated_length": 3947.0, |
| "entropy": 0.2971543218009174, |
| "epoch": 0.292, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.14229293167591095, |
| "kl": 0.0018976893989020027, |
| "learning_rate": 2.016253693614501e-06, |
| "loss": 0.0903, |
| "num_tokens": 16586955.0, |
| "reward": 0.07743750512599945, |
| "reward_std": 0.4277241826057434, |
| "rewards/alfworld_rollout_reward_func/mean": 0.07743750512599945, |
| "rewards/alfworld_rollout_reward_func/std": 0.413460373878479, |
| "sampling/importance_sampling_ratio/max": 2.993452548980713, |
| "sampling/importance_sampling_ratio/mean": 0.56195068359375, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.3662099838256836, |
| "sampling/sampling_logp_difference/mean": 0.01890537701547146, |
| "step": 73, |
| "step_time": 339.4415916220023 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8250.0, |
| "completions/max_terminated_length": 8250.0, |
| "completions/mean_length": 5758.4375, |
| "completions/mean_terminated_length": 5758.4375, |
| "completions/min_length": 1738.0, |
| "completions/min_terminated_length": 1738.0, |
| "entropy": 0.30767737748101354, |
| "epoch": 0.296, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0943857803940773, |
| "kl": 0.0015604560594510986, |
| "learning_rate": 1.9527416904835133e-06, |
| "loss": 0.2749, |
| "num_tokens": 16794329.0, |
| "reward": 0.4194375276565552, |
| "reward_std": 0.572472870349884, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4194375276565552, |
| "rewards/alfworld_rollout_reward_func/std": 0.5662068724632263, |
| "sampling/importance_sampling_ratio/max": 2.8990275859832764, |
| "sampling/importance_sampling_ratio/mean": 0.6088296175003052, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.829563021659851, |
| "sampling/sampling_logp_difference/mean": 0.019225694239139557, |
| "step": 74, |
| "step_time": 292.0825683399162 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6582.0, |
| "completions/max_terminated_length": 6582.0, |
| "completions/mean_length": 3619.1875, |
| "completions/mean_terminated_length": 3619.1875, |
| "completions/min_length": 864.0, |
| "completions/min_terminated_length": 864.0, |
| "entropy": 0.3182159150019288, |
| "epoch": 0.3, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.12715977430343628, |
| "kl": 0.001645655636821175, |
| "learning_rate": 1.889598577429022e-06, |
| "loss": 0.308, |
| "num_tokens": 16929567.0, |
| "reward": 0.6233749985694885, |
| "reward_std": 0.5523929595947266, |
| "rewards/alfworld_rollout_reward_func/mean": 0.6233749985694885, |
| "rewards/alfworld_rollout_reward_func/std": 0.5413455963134766, |
| "sampling/importance_sampling_ratio/max": 2.9408795833587646, |
| "sampling/importance_sampling_ratio/mean": 0.8273114562034607, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.3129551410675049, |
| "sampling/sampling_logp_difference/mean": 0.01962963677942753, |
| "step": 75, |
| "step_time": 207.1375826299918 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5181.0, |
| "completions/max_terminated_length": 5181.0, |
| "completions/mean_length": 4334.53125, |
| "completions/mean_terminated_length": 4334.53125, |
| "completions/min_length": 2063.0, |
| "completions/min_terminated_length": 2063.0, |
| "entropy": 0.34562692465260625, |
| "epoch": 0.304, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08276179432868958, |
| "kl": 0.00154796276910929, |
| "learning_rate": 1.8268669172909137e-06, |
| "loss": 0.1251, |
| "num_tokens": 17084432.0, |
| "reward": 0.1054375022649765, |
| "reward_std": 0.4694763422012329, |
| "rewards/alfworld_rollout_reward_func/mean": 0.1054375022649765, |
| "rewards/alfworld_rollout_reward_func/std": 0.4684988260269165, |
| "sampling/importance_sampling_ratio/max": 1.8063870668411255, |
| "sampling/importance_sampling_ratio/mean": 0.39029014110565186, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.9038729667663574, |
| "sampling/sampling_logp_difference/mean": 0.018809650093317032, |
| "step": 76, |
| "step_time": 208.1383357859595 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10452.0, |
| "completions/max_terminated_length": 10452.0, |
| "completions/mean_length": 8302.25, |
| "completions/mean_terminated_length": 8302.25, |
| "completions/min_length": 2010.0, |
| "completions/min_terminated_length": 2010.0, |
| "entropy": 0.2984990398399532, |
| "epoch": 0.308, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11235751956701279, |
| "kl": 0.0017886007844936103, |
| "learning_rate": 1.7645889955612595e-06, |
| "loss": -0.0816, |
| "num_tokens": 17374744.0, |
| "reward": -0.021937498822808266, |
| "reward_std": 0.22131139039993286, |
| "rewards/alfworld_rollout_reward_func/mean": -0.021937498822808266, |
| "rewards/alfworld_rollout_reward_func/std": 0.28010478615760803, |
| "sampling/importance_sampling_ratio/max": 2.364161252975464, |
| "sampling/importance_sampling_ratio/mean": 0.4629632234573364, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.4188671112060547, |
| "sampling/sampling_logp_difference/mean": 0.01893581636250019, |
| "step": 77, |
| "step_time": 399.66260964999674 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9694.0, |
| "completions/max_terminated_length": 9694.0, |
| "completions/mean_length": 6271.25, |
| "completions/mean_terminated_length": 6271.25, |
| "completions/min_length": 1489.0, |
| "completions/min_terminated_length": 1489.0, |
| "entropy": 0.4018568922765553, |
| "epoch": 0.312, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13200783729553223, |
| "kl": 0.0022579578471777495, |
| "learning_rate": 1.7028067918809539e-06, |
| "loss": -0.2553, |
| "num_tokens": 17594976.0, |
| "reward": 0.2406875193119049, |
| "reward_std": 0.5118597745895386, |
| "rewards/alfworld_rollout_reward_func/mean": 0.2406875193119049, |
| "rewards/alfworld_rollout_reward_func/std": 0.5355828404426575, |
| "sampling/importance_sampling_ratio/max": 2.479753017425537, |
| "sampling/importance_sampling_ratio/mean": 0.3227984309196472, |
| "sampling/importance_sampling_ratio/min": 0.002002666238695383, |
| "sampling/sampling_logp_difference/max": 2.8022069931030273, |
| "sampling/sampling_logp_difference/mean": 0.023829758167266846, |
| "step": 78, |
| "step_time": 299.9485474039684 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6824.0, |
| "completions/max_terminated_length": 6824.0, |
| "completions/mean_length": 4525.46875, |
| "completions/mean_terminated_length": 4525.46875, |
| "completions/min_length": 1223.0, |
| "completions/min_terminated_length": 1223.0, |
| "entropy": 0.35179836209863424, |
| "epoch": 0.316, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09291504323482513, |
| "kl": 0.0013656286282639485, |
| "learning_rate": 1.6415619517425296e-06, |
| "loss": 0.1543, |
| "num_tokens": 17756335.0, |
| "reward": 0.20887500047683716, |
| "reward_std": 0.5421645045280457, |
| "rewards/alfworld_rollout_reward_func/mean": 0.20887500047683716, |
| "rewards/alfworld_rollout_reward_func/std": 0.5290061235427856, |
| "sampling/importance_sampling_ratio/max": 2.4969656467437744, |
| "sampling/importance_sampling_ratio/mean": 0.4944171905517578, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.593350887298584, |
| "sampling/sampling_logp_difference/mean": 0.018796950578689575, |
| "step": 79, |
| "step_time": 253.6868438229867 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6889.0, |
| "completions/max_terminated_length": 6889.0, |
| "completions/mean_length": 5765.34375, |
| "completions/mean_terminated_length": 5765.34375, |
| "completions/min_length": 1729.0, |
| "completions/min_terminated_length": 1729.0, |
| "entropy": 0.29748341953381896, |
| "epoch": 0.32, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07730242609977722, |
| "kl": 0.0013945158752903808, |
| "learning_rate": 1.5808957584181997e-06, |
| "loss": 0.1198, |
| "num_tokens": 17960122.0, |
| "reward": 0.04587499797344208, |
| "reward_std": 0.2844948172569275, |
| "rewards/alfworld_rollout_reward_func/mean": 0.04587499797344208, |
| "rewards/alfworld_rollout_reward_func/std": 0.3961002826690674, |
| "sampling/importance_sampling_ratio/max": 1.659905195236206, |
| "sampling/importance_sampling_ratio/mean": 0.4384477138519287, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.0202646255493164, |
| "sampling/sampling_logp_difference/mean": 0.017428694292902946, |
| "step": 80, |
| "step_time": 264.54027753502305 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5182.0, |
| "completions/max_terminated_length": 5182.0, |
| "completions/mean_length": 4580.71875, |
| "completions/mean_terminated_length": 4580.71875, |
| "completions/min_length": 4178.0, |
| "completions/min_terminated_length": 4178.0, |
| "entropy": 0.40502514131367207, |
| "epoch": 0.324, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.20536404848098755, |
| "kl": 0.0015018332487670705, |
| "learning_rate": 1.5208491051320745e-06, |
| "loss": 0.3905, |
| "num_tokens": 18122737.0, |
| "reward": -0.019312500953674316, |
| "reward_std": 0.2084033489227295, |
| "rewards/alfworld_rollout_reward_func/mean": -0.019312500953674316, |
| "rewards/alfworld_rollout_reward_func/std": 0.261531263589859, |
| "sampling/importance_sampling_ratio/max": 2.869575262069702, |
| "sampling/importance_sampling_ratio/mean": 0.4597967863082886, |
| "sampling/importance_sampling_ratio/min": 0.007556523662060499, |
| "sampling/sampling_logp_difference/max": 0.9722529649734497, |
| "sampling/sampling_logp_difference/mean": 0.020884549245238304, |
| "step": 81, |
| "step_time": 248.1610625180183 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5441.0, |
| "completions/max_terminated_length": 5441.0, |
| "completions/mean_length": 4244.53125, |
| "completions/mean_terminated_length": 4244.53125, |
| "completions/min_length": 774.0, |
| "completions/min_terminated_length": 774.0, |
| "entropy": 0.36706031998619437, |
| "epoch": 0.328, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07920122891664505, |
| "kl": 0.0016353318824258167, |
| "learning_rate": 1.4614624674952843e-06, |
| "loss": 0.0544, |
| "num_tokens": 18276066.0, |
| "reward": 0.327937513589859, |
| "reward_std": 0.5772324204444885, |
| "rewards/alfworld_rollout_reward_func/mean": 0.327937513589859, |
| "rewards/alfworld_rollout_reward_func/std": 0.5514453649520874, |
| "sampling/importance_sampling_ratio/max": 2.0194201469421387, |
| "sampling/importance_sampling_ratio/mean": 0.4743329584598541, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.8554327487945557, |
| "sampling/sampling_logp_difference/mean": 0.02121732383966446, |
| "step": 82, |
| "step_time": 202.4960144519282 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8424.0, |
| "completions/max_terminated_length": 8424.0, |
| "completions/mean_length": 1882.84375, |
| "completions/mean_terminated_length": 1882.84375, |
| "completions/min_length": 867.0, |
| "completions/min_terminated_length": 867.0, |
| "entropy": 0.2702444209717214, |
| "epoch": 0.332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07358190417289734, |
| "kl": 0.0028691932539004483, |
| "learning_rate": 1.402775876222611e-06, |
| "loss": 0.1489, |
| "num_tokens": 18360957.0, |
| "reward": 0.9632500410079956, |
| "reward_std": 0.13971976935863495, |
| "rewards/alfworld_rollout_reward_func/mean": 0.9632500410079956, |
| "rewards/alfworld_rollout_reward_func/std": 0.2857166528701782, |
| "sampling/importance_sampling_ratio/max": 1.8459749221801758, |
| "sampling/importance_sampling_ratio/mean": 0.7306889295578003, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.2139201164245605, |
| "sampling/sampling_logp_difference/mean": 0.018962852656841278, |
| "step": 83, |
| "step_time": 191.13928870702512 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6182.0, |
| "completions/max_terminated_length": 6182.0, |
| "completions/mean_length": 3669.625, |
| "completions/mean_terminated_length": 3669.625, |
| "completions/min_length": 837.0, |
| "completions/min_terminated_length": 837.0, |
| "entropy": 0.3000498446635902, |
| "epoch": 0.336, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.12246961146593094, |
| "kl": 0.0016787616987130605, |
| "learning_rate": 1.3448288901490094e-06, |
| "loss": 0.3255, |
| "num_tokens": 18497233.0, |
| "reward": 0.7534999847412109, |
| "reward_std": 0.39516681432724, |
| "rewards/alfworld_rollout_reward_func/mean": 0.7534999847412109, |
| "rewards/alfworld_rollout_reward_func/std": 0.46645042300224304, |
| "sampling/importance_sampling_ratio/max": 2.5030677318573, |
| "sampling/importance_sampling_ratio/mean": 0.5658935904502869, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.45152747631073, |
| "sampling/sampling_logp_difference/mean": 0.020223423838615417, |
| "step": 84, |
| "step_time": 192.9754896939703 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 4404.0, |
| "completions/max_terminated_length": 4404.0, |
| "completions/mean_length": 3087.5, |
| "completions/mean_terminated_length": 3087.5, |
| "completions/min_length": 454.0, |
| "completions/min_terminated_length": 454.0, |
| "entropy": 0.3710099817253649, |
| "epoch": 0.34, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11732742190361023, |
| "kl": 0.0023317336108448217, |
| "learning_rate": 1.2876605695642086e-06, |
| "loss": 0.1281, |
| "num_tokens": 18611873.0, |
| "reward": 0.4307500123977661, |
| "reward_std": 0.5411006808280945, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4307500123977661, |
| "rewards/alfworld_rollout_reward_func/std": 0.5231669545173645, |
| "sampling/importance_sampling_ratio/max": 2.7108399868011475, |
| "sampling/importance_sampling_ratio/mean": 0.6843256950378418, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.8872215747833252, |
| "sampling/sampling_logp_difference/mean": 0.022719303146004677, |
| "step": 85, |
| "step_time": 162.82808318003663 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10096.0, |
| "completions/max_terminated_length": 10096.0, |
| "completions/mean_length": 8520.375, |
| "completions/mean_terminated_length": 8520.375, |
| "completions/min_length": 3267.0, |
| "completions/min_terminated_length": 3267.0, |
| "entropy": 0.31492479564622045, |
| "epoch": 0.344, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.061374712735414505, |
| "kl": 0.0017404387290298473, |
| "learning_rate": 1.2313094498833612e-06, |
| "loss": 0.1858, |
| "num_tokens": 18910285.0, |
| "reward": 0.15693750977516174, |
| "reward_std": 0.5100167989730835, |
| "rewards/alfworld_rollout_reward_func/mean": 0.15693750977516174, |
| "rewards/alfworld_rollout_reward_func/std": 0.49821487069129944, |
| "sampling/importance_sampling_ratio/max": 1.8992809057235718, |
| "sampling/importance_sampling_ratio/mean": 0.33807021379470825, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.059936761856079, |
| "sampling/sampling_logp_difference/mean": 0.020513102412223816, |
| "step": 86, |
| "step_time": 375.63238986296346 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9171.0, |
| "completions/max_terminated_length": 9171.0, |
| "completions/mean_length": 5972.375, |
| "completions/mean_terminated_length": 5972.375, |
| "completions/min_length": 2094.0, |
| "completions/min_terminated_length": 2094.0, |
| "entropy": 0.3214904679916799, |
| "epoch": 0.348, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0639876201748848, |
| "kl": 0.001944402048138727, |
| "learning_rate": 1.1758135156715043e-06, |
| "loss": 0.2002, |
| "num_tokens": 19126009.0, |
| "reward": 0.542312502861023, |
| "reward_std": 0.5635517239570618, |
| "rewards/alfworld_rollout_reward_func/mean": 0.542312502861023, |
| "rewards/alfworld_rollout_reward_func/std": 0.5554862022399902, |
| "sampling/importance_sampling_ratio/max": 2.4035162925720215, |
| "sampling/importance_sampling_ratio/mean": 0.48470091819763184, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.181654691696167, |
| "sampling/sampling_logp_difference/mean": 0.019984111189842224, |
| "step": 87, |
| "step_time": 301.96421380495303 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6850.0, |
| "completions/max_terminated_length": 6850.0, |
| "completions/mean_length": 6028.90625, |
| "completions/mean_terminated_length": 6028.90625, |
| "completions/min_length": 5420.0, |
| "completions/min_terminated_length": 5420.0, |
| "entropy": 0.3696739445440471, |
| "epoch": 0.352, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.038705118000507355, |
| "kl": 0.0015279942108463729, |
| "learning_rate": 1.1212101750393236e-06, |
| "loss": 0.0118, |
| "num_tokens": 19337782.0, |
| "reward": -0.10300000011920929, |
| "reward_std": 0.132969468832016, |
| "rewards/alfworld_rollout_reward_func/mean": -0.10300000011920929, |
| "rewards/alfworld_rollout_reward_func/std": 0.19296397268772125, |
| "sampling/importance_sampling_ratio/max": 2.475087881088257, |
| "sampling/importance_sampling_ratio/mean": 0.3325284719467163, |
| "sampling/importance_sampling_ratio/min": 0.006633421406149864, |
| "sampling/sampling_logp_difference/max": 2.3745415210723877, |
| "sampling/sampling_logp_difference/mean": 0.01984409987926483, |
| "step": 88, |
| "step_time": 256.6366900990397 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5329.0, |
| "completions/max_terminated_length": 5329.0, |
| "completions/mean_length": 3667.71875, |
| "completions/mean_terminated_length": 3667.71875, |
| "completions/min_length": 1062.0, |
| "completions/min_terminated_length": 1062.0, |
| "entropy": 0.3431046442128718, |
| "epoch": 0.356, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05565686523914337, |
| "kl": 0.002557152631197823, |
| "learning_rate": 1.0675362344274953e-06, |
| "loss": 0.014, |
| "num_tokens": 19471757.0, |
| "reward": 0.44062501192092896, |
| "reward_std": 0.5888760685920715, |
| "rewards/alfworld_rollout_reward_func/mean": 0.44062501192092896, |
| "rewards/alfworld_rollout_reward_func/std": 0.5690953731536865, |
| "sampling/importance_sampling_ratio/max": 2.401775598526001, |
| "sampling/importance_sampling_ratio/mean": 0.42710959911346436, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.14093017578125, |
| "sampling/sampling_logp_difference/mean": 0.022225676104426384, |
| "step": 89, |
| "step_time": 193.9669523180055 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6081.0, |
| "completions/max_terminated_length": 6081.0, |
| "completions/mean_length": 4530.09375, |
| "completions/mean_terminated_length": 4530.09375, |
| "completions/min_length": 3277.0, |
| "completions/min_terminated_length": 3277.0, |
| "entropy": 0.4193602460436523, |
| "epoch": 0.36, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.15843002498149872, |
| "kl": 0.0020286712642700877, |
| "learning_rate": 1.0148278737965845e-06, |
| "loss": -0.0643, |
| "num_tokens": 19632336.0, |
| "reward": 0.12999999523162842, |
| "reward_std": 0.4567090570926666, |
| "rewards/alfworld_rollout_reward_func/mean": 0.12999999523162842, |
| "rewards/alfworld_rollout_reward_func/std": 0.45741432905197144, |
| "sampling/importance_sampling_ratio/max": 2.779259204864502, |
| "sampling/importance_sampling_ratio/mean": 0.5273926854133606, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.931565523147583, |
| "sampling/sampling_logp_difference/mean": 0.02205650880932808, |
| "step": 90, |
| "step_time": 242.78942537099647 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9883.0, |
| "completions/max_terminated_length": 9883.0, |
| "completions/mean_length": 6145.40625, |
| "completions/mean_terminated_length": 6145.40625, |
| "completions/min_length": 2700.0, |
| "completions/min_terminated_length": 2700.0, |
| "entropy": 0.30875492095947266, |
| "epoch": 0.364, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1301383525133133, |
| "kl": 0.0016387706618843367, |
| "learning_rate": 9.63120622239248e-07, |
| "loss": 0.2844, |
| "num_tokens": 19851965.0, |
| "reward": 0.4439375102519989, |
| "reward_std": 0.5609219074249268, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4439375102519989, |
| "rewards/alfworld_rollout_reward_func/std": 0.5704147815704346, |
| "sampling/importance_sampling_ratio/max": 2.949549436569214, |
| "sampling/importance_sampling_ratio/mean": 0.6360998153686523, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.2484278678894043, |
| "sampling/sampling_logp_difference/mean": 0.018874069675803185, |
| "step": 91, |
| "step_time": 324.5515782940056 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 4739.0, |
| "completions/max_terminated_length": 4739.0, |
| "completions/mean_length": 3070.40625, |
| "completions/mean_terminated_length": 3070.40625, |
| "completions/min_length": 691.0, |
| "completions/min_terminated_length": 691.0, |
| "entropy": 0.4392261947505176, |
| "epoch": 0.368, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07320296764373779, |
| "kl": 0.002107658132445067, |
| "learning_rate": 9.124493340311538e-07, |
| "loss": 0.2317, |
| "num_tokens": 19964394.0, |
| "reward": 0.25699999928474426, |
| "reward_std": 0.5001701712608337, |
| "rewards/alfworld_rollout_reward_func/mean": 0.25699999928474426, |
| "rewards/alfworld_rollout_reward_func/std": 0.5264375805854797, |
| "sampling/importance_sampling_ratio/max": 1.6854748725891113, |
| "sampling/importance_sampling_ratio/mean": 0.3533988893032074, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7995140552520752, |
| "sampling/sampling_logp_difference/mean": 0.025006595999002457, |
| "step": 92, |
| "step_time": 175.92185580494697 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6947.0, |
| "completions/max_terminated_length": 6947.0, |
| "completions/mean_length": 3786.75, |
| "completions/mean_terminated_length": 3786.75, |
| "completions/min_length": 724.0, |
| "completions/min_terminated_length": 724.0, |
| "entropy": 0.2946209688670933, |
| "epoch": 0.372, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08050260692834854, |
| "kl": 0.0019706511029653484, |
| "learning_rate": 8.628481651367876e-07, |
| "loss": 0.1288, |
| "num_tokens": 20106562.0, |
| "reward": 0.6368750333786011, |
| "reward_std": 0.5256940126419067, |
| "rewards/alfworld_rollout_reward_func/mean": 0.6368750333786011, |
| "rewards/alfworld_rollout_reward_func/std": 0.5401507616043091, |
| "sampling/importance_sampling_ratio/max": 2.974215507507324, |
| "sampling/importance_sampling_ratio/mean": 0.6188757419586182, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.8094772100448608, |
| "sampling/sampling_logp_difference/mean": 0.020193612203001976, |
| "step": 93, |
| "step_time": 206.48572037101258 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5326.0, |
| "completions/max_terminated_length": 5326.0, |
| "completions/mean_length": 4152.09375, |
| "completions/mean_terminated_length": 4152.09375, |
| "completions/min_length": 2084.0, |
| "completions/min_terminated_length": 2084.0, |
| "entropy": 0.4083614726550877, |
| "epoch": 0.376, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07430887222290039, |
| "kl": 0.0020167900111118797, |
| "learning_rate": 8.143505501859553e-07, |
| "loss": 0.1415, |
| "num_tokens": 20254149.0, |
| "reward": -0.057624995708465576, |
| "reward_std": 0.22674137353897095, |
| "rewards/alfworld_rollout_reward_func/mean": -0.057624995708465576, |
| "rewards/alfworld_rollout_reward_func/std": 0.28084638714790344, |
| "sampling/importance_sampling_ratio/max": 1.6986135244369507, |
| "sampling/importance_sampling_ratio/mean": 0.3223665952682495, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.5820741653442383, |
| "sampling/sampling_logp_difference/mean": 0.022008778527379036, |
| "step": 94, |
| "step_time": 228.1358192209882 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11112.0, |
| "completions/max_terminated_length": 11112.0, |
| "completions/mean_length": 7901.9375, |
| "completions/mean_terminated_length": 7901.9375, |
| "completions/min_length": 2256.0, |
| "completions/min_terminated_length": 2256.0, |
| "entropy": 0.3604189087636769, |
| "epoch": 0.38, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11994580924510956, |
| "kl": 0.002103939070366323, |
| "learning_rate": 7.669891799365284e-07, |
| "loss": 0.2477, |
| "num_tokens": 20531587.0, |
| "reward": 0.1081249937415123, |
| "reward_std": 0.3899664282798767, |
| "rewards/alfworld_rollout_reward_func/mean": 0.1081249937415123, |
| "rewards/alfworld_rollout_reward_func/std": 0.44592028856277466, |
| "sampling/importance_sampling_ratio/max": 2.60292649269104, |
| "sampling/importance_sampling_ratio/mean": 0.4245090186595917, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.6102848052978516, |
| "sampling/sampling_logp_difference/mean": 0.023151205852627754, |
| "step": 95, |
| "step_time": 380.974294352025 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 6028.0, |
| "completions/max_terminated_length": 6028.0, |
| "completions/mean_length": 3872.59375, |
| "completions/mean_terminated_length": 3872.59375, |
| "completions/min_length": 1222.0, |
| "completions/min_terminated_length": 1222.0, |
| "entropy": 0.2878755168057978, |
| "epoch": 0.384, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09124290198087692, |
| "kl": 0.0014382937183654576, |
| "learning_rate": 7.207959792385999e-07, |
| "loss": 0.4866, |
| "num_tokens": 20673142.0, |
| "reward": 0.390375018119812, |
| "reward_std": 0.6189597845077515, |
| "rewards/alfworld_rollout_reward_func/mean": 0.390375018119812, |
| "rewards/alfworld_rollout_reward_func/std": 0.5916039347648621, |
| "sampling/importance_sampling_ratio/max": 1.520223617553711, |
| "sampling/importance_sampling_ratio/mean": 0.46078407764434814, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7394518852233887, |
| "sampling/sampling_logp_difference/mean": 0.017645301297307014, |
| "step": 96, |
| "step_time": 206.5706790599943 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13788.0, |
| "completions/max_terminated_length": 13788.0, |
| "completions/mean_length": 10405.34375, |
| "completions/mean_terminated_length": 10405.34375, |
| "completions/min_length": 1455.0, |
| "completions/min_terminated_length": 1455.0, |
| "entropy": 0.2904400497209281, |
| "epoch": 0.388, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06394726037979126, |
| "kl": 0.002259159025925328, |
| "learning_rate": 6.75802085514925e-07, |
| "loss": 0.1384, |
| "num_tokens": 21039201.0, |
| "reward": 0.15262500941753387, |
| "reward_std": 0.3974687457084656, |
| "rewards/alfworld_rollout_reward_func/mean": 0.15262500941753387, |
| "rewards/alfworld_rollout_reward_func/std": 0.4503324627876282, |
| "sampling/importance_sampling_ratio/max": 1.852544903755188, |
| "sampling/importance_sampling_ratio/mean": 0.4211999773979187, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.9718213081359863, |
| "sampling/sampling_logp_difference/mean": 0.020324187353253365, |
| "step": 97, |
| "step_time": 493.7330398180784 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9815.0, |
| "completions/max_terminated_length": 9815.0, |
| "completions/mean_length": 8616.3125, |
| "completions/mean_terminated_length": 8616.3125, |
| "completions/min_length": 5394.0, |
| "completions/min_terminated_length": 5394.0, |
| "entropy": 0.29368510097265244, |
| "epoch": 0.392, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.053294822573661804, |
| "kl": 0.002172048004467797, |
| "learning_rate": 6.320378277721342e-07, |
| "loss": 0.0355, |
| "num_tokens": 21339499.0, |
| "reward": -0.05612500011920929, |
| "reward_std": 0.12973152101039886, |
| "rewards/alfworld_rollout_reward_func/mean": -0.05612500011920929, |
| "rewards/alfworld_rollout_reward_func/std": 0.1938219666481018, |
| "sampling/importance_sampling_ratio/max": 2.012138843536377, |
| "sampling/importance_sampling_ratio/mean": 0.4920834004878998, |
| "sampling/importance_sampling_ratio/min": 5.605193857299268e-45, |
| "sampling/sampling_logp_difference/max": 47.1961555480957, |
| "sampling/sampling_logp_difference/mean": 0.026191767305135727, |
| "step": 98, |
| "step_time": 378.82437252601085 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5773.0, |
| "completions/max_terminated_length": 5773.0, |
| "completions/mean_length": 4738.5, |
| "completions/mean_terminated_length": 4738.5, |
| "completions/min_length": 1107.0, |
| "completions/min_terminated_length": 1107.0, |
| "entropy": 0.3817450860515237, |
| "epoch": 0.396, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13572344183921814, |
| "kl": 0.001779558529960923, |
| "learning_rate": 5.895327061568776e-07, |
| "loss": 0.1925, |
| "num_tokens": 21508315.0, |
| "reward": 0.09443750977516174, |
| "reward_std": 0.3808988630771637, |
| "rewards/alfworld_rollout_reward_func/mean": 0.09443750977516174, |
| "rewards/alfworld_rollout_reward_func/std": 0.43921801447868347, |
| "sampling/importance_sampling_ratio/max": 2.9193501472473145, |
| "sampling/importance_sampling_ratio/mean": 0.6009063720703125, |
| "sampling/importance_sampling_ratio/min": 0.01660267822444439, |
| "sampling/sampling_logp_difference/max": 1.1129672527313232, |
| "sampling/sampling_logp_difference/mean": 0.02005705237388611, |
| "step": 99, |
| "step_time": 232.97595874097897 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7291.0, |
| "completions/max_terminated_length": 7291.0, |
| "completions/mean_length": 5083.0625, |
| "completions/mean_terminated_length": 5083.0625, |
| "completions/min_length": 1140.0, |
| "completions/min_terminated_length": 1140.0, |
| "entropy": 0.34929203847423196, |
| "epoch": 0.4, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13283155858516693, |
| "kl": 0.0015651150179110118, |
| "learning_rate": 5.483153720706799e-07, |
| "loss": -0.0391, |
| "num_tokens": 21690781.0, |
| "reward": 0.4309374690055847, |
| "reward_std": 0.5609171390533447, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4309374690055847, |
| "rewards/alfworld_rollout_reward_func/std": 0.5701945424079895, |
| "sampling/importance_sampling_ratio/max": 2.6785364151000977, |
| "sampling/importance_sampling_ratio/mean": 0.6643563508987427, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.5328056812286377, |
| "sampling/sampling_logp_difference/mean": 0.020073113963007927, |
| "step": 100, |
| "step_time": 262.673913760038 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 125, |
| "num_input_tokens_seen": 21690781, |
| "num_train_epochs": 1, |
| "save_steps": 25, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
