diff --git "a/checkpoint-64/trainer_state.json" "b/checkpoint-64/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-64/trainer_state.json" @@ -0,0 +1,2466 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.512, + "eval_steps": 500, + "global_step": 64, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 1707.0, + "completions/max_terminated_length": 915.0, + "completions/mean_length": 812.703125, + "completions/mean_terminated_length": 386.1499938964844, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.319809939712286, + "epoch": 0.008, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013934657908976078, + "kl": 0.0, + "learning_rate": 0.0, + "loss": 0.0003, + "num_tokens": 52013.0, + "reward": 0.26906251907348633, + "reward_std": 0.43214160203933716, + "rewards/alfworld_rollout_reward_func/mean": 0.26906251907348633, + "rewards/alfworld_rollout_reward_func/std": 0.48894354701042175, + "rollout/action_tokens_mean": 812.703125, + "rollout/episode_turns_mean": 25.0625, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.6875, + "sampling/importance_sampling_ratio/max": 2.401656150817871, + "sampling/importance_sampling_ratio/mean": 0.5413336753845215, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.9913136959075928, + "sampling/sampling_logp_difference/mean": 0.020846465602517128, + "step": 1, + "step_time": 340.2447626780008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.765625, + "completions/max_length": 1338.0, + "completions/max_terminated_length": 912.0, + "completions/mean_length": 822.90625, + "completions/mean_terminated_length": 481.0000305175781, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.33951753890141845, + "epoch": 0.016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005984712392091751, + "kl": 0.0, + "learning_rate": 2.5e-06, + "loss": 0.0064, + "num_tokens": 104679.0, + "reward": 0.1837500035762787, + "reward_std": 0.37211528420448303, + "rewards/alfworld_rollout_reward_func/mean": 0.1837500035762787, + "rewards/alfworld_rollout_reward_func/std": 0.44909608364105225, + "rollout/action_tokens_mean": 822.90625, + "rollout/episode_turns_mean": 26.75, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.765625, + "sampling/importance_sampling_ratio/max": 2.0904664993286133, + "sampling/importance_sampling_ratio/mean": 0.30532318353652954, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.2203328609466553, + "sampling/sampling_logp_difference/mean": 0.021448608487844467, + "step": 2, + "step_time": 376.3160819019986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 1253.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 735.65625, + "completions/mean_terminated_length": 380.5, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "entropy": 0.32836992759257555, + "epoch": 0.024, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0066122072748839855, + "kl": 0.001546632844110718, + "learning_rate": 5e-06, + "loss": 0.0026, + "num_tokens": 151761.0, + "reward": 0.21156249940395355, + "reward_std": 0.2755644917488098, + "rewards/alfworld_rollout_reward_func/mean": 0.21156249940395355, + "rewards/alfworld_rollout_reward_func/std": 0.44746631383895874, + "rollout/action_tokens_mean": 735.65625, + "rollout/episode_turns_mean": 26.03125, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.75, + "sampling/importance_sampling_ratio/max": 2.4706838130950928, + "sampling/importance_sampling_ratio/mean": 0.5484369993209839, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.5531697273254395, + "sampling/sampling_logp_difference/mean": 0.020412152633070946, + "step": 3, + "step_time": 359.0840177950013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.546875, + "completions/max_length": 1880.0, + "completions/max_terminated_length": 807.0, + "completions/mean_length": 721.78125, + "completions/mean_terminated_length": 425.72412109375, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "entropy": 0.3564013452269137, + "epoch": 0.032, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00901270005851984, + "kl": 0.001597001251866459, + "learning_rate": 4.996791267927632e-06, + "loss": 0.0123, + "num_tokens": 197955.0, + "reward": 0.40953126549720764, + "reward_std": 0.48288512229919434, + "rewards/alfworld_rollout_reward_func/mean": 0.40953126549720764, + "rewards/alfworld_rollout_reward_func/std": 0.5298889875411987, + "rollout/action_tokens_mean": 721.78125, + "rollout/episode_turns_mean": 22.859375, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.546875, + "sampling/importance_sampling_ratio/max": 2.607266664505005, + "sampling/importance_sampling_ratio/mean": 0.5041748881340027, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.2973906993865967, + "sampling/sampling_logp_difference/mean": 0.022099550813436508, + "step": 4, + "step_time": 324.93751245399835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 1772.0, + "completions/max_terminated_length": 1135.0, + "completions/mean_length": 823.984375, + "completions/mean_terminated_length": 511.25, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.37002173252403736, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006255371496081352, + "kl": 0.0018659899324120488, + "learning_rate": 4.987173308479738e-06, + "loss": -0.0049, + "num_tokens": 250690.0, + "reward": 0.2629687488079071, + "reward_std": 0.3148724138736725, + "rewards/alfworld_rollout_reward_func/mean": 0.2629687786102295, + "rewards/alfworld_rollout_reward_func/std": 0.48142871260643005, + "rollout/action_tokens_mean": 823.984375, + "rollout/episode_turns_mean": 25.65625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.6875, + "sampling/importance_sampling_ratio/max": 2.4376842975616455, + "sampling/importance_sampling_ratio/mean": 0.3966376781463623, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.3790063858032227, + "sampling/sampling_logp_difference/mean": 0.022958241403102875, + "step": 5, + "step_time": 428.7077085350011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.546875, + "completions/max_length": 1754.0, + "completions/max_terminated_length": 1081.0, + "completions/mean_length": 778.265625, + "completions/mean_terminated_length": 513.0689697265625, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "entropy": 0.33683515526354313, + "epoch": 0.048, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014856722205877304, + "kl": 0.0018188257581641665, + "learning_rate": 4.971170810820279e-06, + "loss": 0.0284, + "num_tokens": 300499.0, + "reward": 0.4112499952316284, + "reward_std": 0.4292137622833252, + "rewards/alfworld_rollout_reward_func/mean": 0.4112499952316284, + "rewards/alfworld_rollout_reward_func/std": 0.5246087312698364, + "rollout/action_tokens_mean": 778.265625, + "rollout/episode_turns_mean": 24.0, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.546875, + "sampling/importance_sampling_ratio/max": 2.469376802444458, + "sampling/importance_sampling_ratio/mean": 0.41494274139404297, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.5756630897521973, + "sampling/sampling_logp_difference/mean": 0.020635541528463364, + "step": 6, + "step_time": 329.3498830619992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.765625, + "completions/max_length": 1750.0, + "completions/max_terminated_length": 882.0, + "completions/mean_length": 834.359375, + "completions/mean_terminated_length": 538.4000244140625, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "entropy": 0.37804630119353533, + "epoch": 0.056, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011092925444245338, + "kl": 0.0017349459521938115, + "learning_rate": 4.948824853131237e-06, + "loss": 0.0135, + "num_tokens": 353898.0, + "reward": 0.1875, + "reward_std": 0.41170334815979004, + "rewards/alfworld_rollout_reward_func/mean": 0.1875, + "rewards/alfworld_rollout_reward_func/std": 0.4403570294380188, + "rollout/action_tokens_mean": 834.359375, + "rollout/episode_turns_mean": 27.0625, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.765625, + "sampling/importance_sampling_ratio/max": 2.2279603481292725, + "sampling/importance_sampling_ratio/mean": 0.46820664405822754, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.733177900314331, + "sampling/sampling_logp_difference/mean": 0.022584954276680946, + "step": 7, + "step_time": 372.10799923799914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.765625, + "completions/max_length": 1682.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 786.5625, + "completions/mean_terminated_length": 380.8000183105469, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "entropy": 0.32334376219660044, + "epoch": 0.064, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010468119755387306, + "kl": 0.0017176041819766397, + "learning_rate": 4.920192797165511e-06, + "loss": -0.0059, + "num_tokens": 404238.0, + "reward": 0.1939062476158142, + "reward_std": 0.31658345460891724, + "rewards/alfworld_rollout_reward_func/mean": 0.1939062476158142, + "rewards/alfworld_rollout_reward_func/std": 0.44521304965019226, + "rollout/action_tokens_mean": 786.5625, + "rollout/episode_turns_mean": 25.984375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.765625, + "sampling/importance_sampling_ratio/max": 2.8437609672546387, + "sampling/importance_sampling_ratio/mean": 0.6038376092910767, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.8486413955688477, + "sampling/sampling_logp_difference/mean": 0.0203598253428936, + "step": 8, + "step_time": 434.73393120799847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 1492.0, + "completions/max_terminated_length": 1027.0, + "completions/mean_length": 818.34375, + "completions/mean_terminated_length": 474.0625, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 0.32139473129063845, + "epoch": 0.072, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006268487311899662, + "kl": 0.0026504553206905257, + "learning_rate": 4.8853481410001225e-06, + "loss": -0.0089, + "num_tokens": 456612.0, + "reward": 0.19500000774860382, + "reward_std": 0.29811495542526245, + "rewards/alfworld_rollout_reward_func/mean": 0.19500000774860382, + "rewards/alfworld_rollout_reward_func/std": 0.46079298853874207, + "rollout/action_tokens_mean": 818.34375, + "rollout/episode_turns_mean": 26.109375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.75, + "sampling/importance_sampling_ratio/max": 2.6596250534057617, + "sampling/importance_sampling_ratio/mean": 0.42990007996559143, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.3629698753356934, + "sampling/sampling_logp_difference/mean": 0.019470183178782463, + "step": 9, + "step_time": 399.7356371579963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 1381.0, + "completions/max_terminated_length": 1015.0, + "completions/mean_length": 710.296875, + "completions/mean_terminated_length": 485.5384826660156, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "entropy": 0.34738565888255835, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010129230096936226, + "kl": 0.0015055066687637009, + "learning_rate": 4.844380330367701e-06, + "loss": 0.0042, + "num_tokens": 502071.0, + "reward": 0.3684375286102295, + "reward_std": 0.3336261808872223, + "rewards/alfworld_rollout_reward_func/mean": 0.3684375286102295, + "rewards/alfworld_rollout_reward_func/std": 0.5042713284492493, + "rollout/action_tokens_mean": 710.296875, + "rollout/episode_turns_mean": 24.859375, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.59375, + "sampling/importance_sampling_ratio/max": 2.386782646179199, + "sampling/importance_sampling_ratio/mean": 0.5169863104820251, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.2723636627197266, + "sampling/sampling_logp_difference/mean": 0.022041790187358856, + "step": 10, + "step_time": 394.5470729360013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.859375, + "completions/max_length": 1352.0, + "completions/max_terminated_length": 782.0, + "completions/mean_length": 881.84375, + "completions/mean_terminated_length": 523.4444580078125, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "entropy": 0.3616171991452575, + "epoch": 0.088, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00803722906857729, + "kl": 0.0017023698201228399, + "learning_rate": 4.797394529050577e-06, + "loss": 0.0111, + "num_tokens": 558509.0, + "reward": 0.08921875059604645, + "reward_std": 0.2777005434036255, + "rewards/alfworld_rollout_reward_func/mean": 0.08921875059604645, + "rewards/alfworld_rollout_reward_func/std": 0.36485835909843445, + "rollout/action_tokens_mean": 881.84375, + "rollout/episode_turns_mean": 28.40625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.859375, + "sampling/importance_sampling_ratio/max": 2.7719504833221436, + "sampling/importance_sampling_ratio/mean": 0.4752790629863739, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.038581132888794, + "sampling/sampling_logp_difference/mean": 0.02142297849059105, + "step": 11, + "step_time": 396.1487058620023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.671875, + "completions/max_length": 2292.0, + "completions/max_terminated_length": 901.0, + "completions/mean_length": 860.28125, + "completions/mean_terminated_length": 460.76190185546875, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.35516145126894116, + "epoch": 0.096, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011235510930418968, + "kl": 0.0016268756971840048, + "learning_rate": 4.744511348926855e-06, + "loss": 0.0048, + "num_tokens": 613567.0, + "reward": 0.26890623569488525, + "reward_std": 0.41516801714897156, + "rewards/alfworld_rollout_reward_func/mean": 0.26890626549720764, + "rewards/alfworld_rollout_reward_func/std": 0.4936746656894684, + "rollout/action_tokens_mean": 860.28125, + "rollout/episode_turns_mean": 25.484375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.671875, + "sampling/importance_sampling_ratio/max": 2.7596683502197266, + "sampling/importance_sampling_ratio/mean": 0.5103189945220947, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.6376895904541016, + "sampling/sampling_logp_difference/mean": 0.02156790718436241, + "step": 12, + "step_time": 421.574981908002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.8125, + "completions/max_length": 1382.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 836.53125, + "completions/mean_terminated_length": 586.4166870117188, + "completions/min_length": 402.0, + "completions/min_terminated_length": 402.0, + "entropy": 0.35410969890654087, + "epoch": 0.104, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010335148312151432, + "kl": 0.0018077169770549517, + "learning_rate": 4.685866540361456e-06, + "loss": 0.0068, + "num_tokens": 667105.0, + "reward": 0.14281249046325684, + "reward_std": 0.3198840916156769, + "rewards/alfworld_rollout_reward_func/mean": 0.14281249046325684, + "rewards/alfworld_rollout_reward_func/std": 0.3984073996543884, + "rollout/action_tokens_mean": 836.53125, + "rollout/episode_turns_mean": 28.453125, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.8125, + "sampling/importance_sampling_ratio/max": 2.6870784759521484, + "sampling/importance_sampling_ratio/mean": 0.6008636951446533, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.14093017578125, + "sampling/sampling_logp_difference/mean": 0.021617209538817406, + "step": 13, + "step_time": 369.507772967002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 1584.0, + "completions/max_terminated_length": 1063.0, + "completions/mean_length": 793.703125, + "completions/mean_terminated_length": 525.4500122070312, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "entropy": 0.34002805408090353, + "epoch": 0.112, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00711849657818675, + "kl": 0.0019163258293701801, + "learning_rate": 4.621610643736878e-06, + "loss": 0.0062, + "num_tokens": 717902.0, + "reward": 0.2626562714576721, + "reward_std": 0.3680095076560974, + "rewards/alfworld_rollout_reward_func/mean": 0.26265624165534973, + "rewards/alfworld_rollout_reward_func/std": 0.489940881729126, + "rollout/action_tokens_mean": 793.703125, + "rollout/episode_turns_mean": 26.28125, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.6875, + "sampling/importance_sampling_ratio/max": 2.815065860748291, + "sampling/importance_sampling_ratio/mean": 0.45154738426208496, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.7529265880584717, + "sampling/sampling_logp_difference/mean": 0.0205212514847517, + "step": 14, + "step_time": 327.6827599140015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.84375, + "completions/max_length": 1814.0, + "completions/max_terminated_length": 890.0, + "completions/mean_length": 909.109375, + "completions/mean_terminated_length": 528.7999877929688, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.35556588135659695, + "epoch": 0.12, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010462122038006783, + "kl": 0.0016676104332873365, + "learning_rate": 4.551908603018191e-06, + "loss": -0.0076, + "num_tokens": 776085.0, + "reward": 0.10140626132488251, + "reward_std": 0.36953580379486084, + "rewards/alfworld_rollout_reward_func/mean": 0.10140625387430191, + "rewards/alfworld_rollout_reward_func/std": 0.38891327381134033, + "rollout/action_tokens_mean": 909.109375, + "rollout/episode_turns_mean": 28.0, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.84375, + "sampling/importance_sampling_ratio/max": 2.0437681674957275, + "sampling/importance_sampling_ratio/mean": 0.3665631115436554, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 4.642125606536865, + "sampling/sampling_logp_difference/mean": 0.021498428657650948, + "step": 15, + "step_time": 409.45475878100024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.703125, + "completions/max_length": 2069.0, + "completions/max_terminated_length": 815.0, + "completions/mean_length": 858.484375, + "completions/mean_terminated_length": 363.0, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "entropy": 0.3426384939812124, + "epoch": 0.128, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007739651948213577, + "kl": 0.0015719706807431066, + "learning_rate": 4.476939342344246e-06, + "loss": -0.002, + "num_tokens": 831028.0, + "reward": 0.2515625059604645, + "reward_std": 0.4047408103942871, + "rewards/alfworld_rollout_reward_func/mean": 0.2515625059604645, + "rewards/alfworld_rollout_reward_func/std": 0.48498478531837463, + "rollout/action_tokens_mean": 858.484375, + "rollout/episode_turns_mean": 25.015625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.703125, + "sampling/importance_sampling_ratio/max": 2.197270631790161, + "sampling/importance_sampling_ratio/mean": 0.3564271330833435, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.3733601570129395, + "sampling/sampling_logp_difference/mean": 0.021193569526076317, + "step": 16, + "step_time": 341.3529842470016 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.765625, + "completions/max_length": 1344.0, + "completions/max_terminated_length": 872.0, + "completions/mean_length": 800.890625, + "completions/mean_terminated_length": 410.0000305175781, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "entropy": 0.3335589300841093, + "epoch": 0.136, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4890336990356445, + "kl": 0.8330444995917787, + "learning_rate": 4.396895306731978e-06, + "loss": 0.0005, + "num_tokens": 882285.0, + "reward": 0.18125000596046448, + "reward_std": 0.24295468628406525, + "rewards/alfworld_rollout_reward_func/mean": 0.18124999105930328, + "rewards/alfworld_rollout_reward_func/std": 0.4482647478580475, + "rollout/action_tokens_mean": 800.890625, + "rollout/episode_turns_mean": 26.265625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.765625, + "sampling/importance_sampling_ratio/max": 2.3350934982299805, + "sampling/importance_sampling_ratio/mean": 0.37133079767227173, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 10.240796089172363, + "sampling/sampling_logp_difference/mean": 0.020599324256181717, + "step": 17, + "step_time": 436.20165634099976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.890625, + "completions/max_length": 1359.0, + "completions/max_terminated_length": 872.0, + "completions/mean_length": 864.640625, + "completions/mean_terminated_length": 638.7142944335938, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.3253911091014743, + "epoch": 0.144, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007197697646915913, + "kl": 0.0016915635187615408, + "learning_rate": 4.3119819680728e-06, + "loss": -0.0099, + "num_tokens": 937622.0, + "reward": 0.06968750059604645, + "reward_std": 0.19933855533599854, + "rewards/alfworld_rollout_reward_func/mean": 0.06968750059604645, + "rewards/alfworld_rollout_reward_func/std": 0.3224163353443146, + "rollout/action_tokens_mean": 864.640625, + "rollout/episode_turns_mean": 28.9375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.890625, + "sampling/importance_sampling_ratio/max": 2.2717995643615723, + "sampling/importance_sampling_ratio/mean": 0.5041627287864685, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.0254077911376953, + "sampling/sampling_logp_difference/mean": 0.019858187064528465, + "step": 18, + "step_time": 437.1889309570033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.703125, + "completions/max_length": 2201.0, + "completions/max_terminated_length": 1115.0, + "completions/mean_length": 844.5, + "completions/mean_terminated_length": 540.631591796875, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "entropy": 0.33306339103728533, + "epoch": 0.152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010224551893770695, + "kl": 0.001574573227117071, + "learning_rate": 4.222417297689217e-06, + "loss": 0.0213, + "num_tokens": 991670.0, + "reward": 0.24437499046325684, + "reward_std": 0.42425790429115295, + "rewards/alfworld_rollout_reward_func/mean": 0.24437500536441803, + "rewards/alfworld_rollout_reward_func/std": 0.48038968443870544, + "rollout/action_tokens_mean": 844.5, + "rollout/episode_turns_mean": 26.171875, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.703125, + "sampling/importance_sampling_ratio/max": 2.5866730213165283, + "sampling/importance_sampling_ratio/mean": 0.5376668572425842, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.632045865058899, + "sampling/sampling_logp_difference/mean": 0.01941317319869995, + "step": 19, + "step_time": 348.9997736020032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 1764.0, + "completions/max_terminated_length": 1085.0, + "completions/mean_length": 937.640625, + "completions/mean_terminated_length": 530.6875, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.3477782104164362, + "epoch": 0.16, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010612000711262226, + "kl": 0.001602184345756541, + "learning_rate": 4.128431206805556e-06, + "loss": 0.0044, + "num_tokens": 1051679.0, + "reward": 0.1901562511920929, + "reward_std": 0.39348262548446655, + "rewards/alfworld_rollout_reward_func/mean": 0.1901562511920929, + "rewards/alfworld_rollout_reward_func/std": 0.4530303478240967, + "rollout/action_tokens_mean": 937.640625, + "rollout/episode_turns_mean": 26.6875, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.75, + "sampling/importance_sampling_ratio/max": 2.5002288818359375, + "sampling/importance_sampling_ratio/mean": 0.4413934350013733, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.34731388092041, + "sampling/sampling_logp_difference/mean": 0.0198683924973011, + "step": 20, + "step_time": 353.7044870629943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 1475.0, + "completions/max_terminated_length": 939.0, + "completions/mean_length": 800.453125, + "completions/mean_terminated_length": 418.0909118652344, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.36278699059039354, + "epoch": 0.168, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010538731701672077, + "kl": 0.001873640143458033, + "learning_rate": 4.030264956369158e-06, + "loss": -0.0018, + "num_tokens": 1102908.0, + "reward": 0.2993749976158142, + "reward_std": 0.40504124760627747, + "rewards/alfworld_rollout_reward_func/mean": 0.2993749976158142, + "rewards/alfworld_rollout_reward_func/std": 0.4982314109802246, + "rollout/action_tokens_mean": 800.453125, + "rollout/episode_turns_mean": 24.4375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.65625, + "sampling/importance_sampling_ratio/max": 2.447692632675171, + "sampling/importance_sampling_ratio/mean": 0.448123574256897, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.2475838661193848, + "sampling/sampling_logp_difference/mean": 0.022293493151664734, + "step": 21, + "step_time": 452.2219315199982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 1566.0, + "completions/max_terminated_length": 929.0, + "completions/mean_length": 743.703125, + "completions/mean_terminated_length": 460.91668701171875, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.3329707160592079, + "epoch": 0.176, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007643942255526781, + "kl": 0.0019213874111301266, + "learning_rate": 3.9281705377369814e-06, + "loss": 0.0059, + "num_tokens": 1150505.0, + "reward": 0.32625001668930054, + "reward_std": 0.4726526141166687, + "rewards/alfworld_rollout_reward_func/mean": 0.32624998688697815, + "rewards/alfworld_rollout_reward_func/std": 0.5121089816093445, + "rollout/action_tokens_mean": 743.703125, + "rollout/episode_turns_mean": 24.796875, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.625, + "sampling/importance_sampling_ratio/max": 2.206357479095459, + "sampling/importance_sampling_ratio/mean": 0.4393410086631775, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.366598129272461, + "sampling/sampling_logp_difference/mean": 0.020861441269516945, + "step": 22, + "step_time": 350.210106904995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 1494.0, + "completions/max_terminated_length": 1159.0, + "completions/mean_length": 797.828125, + "completions/mean_terminated_length": 504.04547119140625, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "entropy": 0.35823060059919953, + "epoch": 0.184, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010937543585896492, + "kl": 0.00202080328745069, + "learning_rate": 3.8224100258174066e-06, + "loss": 0.0064, + "num_tokens": 1201566.0, + "reward": 0.3057812452316284, + "reward_std": 0.36121153831481934, + "rewards/alfworld_rollout_reward_func/mean": 0.3057812452316284, + "rewards/alfworld_rollout_reward_func/std": 0.49343568086624146, + "rollout/action_tokens_mean": 797.828125, + "rollout/episode_turns_mean": 25.328125, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.65625, + "sampling/importance_sampling_ratio/max": 2.8112552165985107, + "sampling/importance_sampling_ratio/mean": 0.43328696489334106, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.2014861106872559, + "sampling/sampling_logp_difference/mean": 0.022007007151842117, + "step": 23, + "step_time": 299.4183257309978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 1409.0, + "completions/max_terminated_length": 962.0, + "completions/mean_length": 749.84375, + "completions/mean_terminated_length": 406.0, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "entropy": 0.314591265283525, + "epoch": 0.192, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008087356574833393, + "kl": 0.0017565299822308589, + "learning_rate": 3.7132549063277033e-06, + "loss": -0.0022, + "num_tokens": 1249556.0, + "reward": 0.20937500894069672, + "reward_std": 0.2735254764556885, + "rewards/alfworld_rollout_reward_func/mean": 0.20937500894069672, + "rewards/alfworld_rollout_reward_func/std": 0.45049840211868286, + "rollout/action_tokens_mean": 749.84375, + "rollout/episode_turns_mean": 25.53125, + "rollout/mismatch_mean": 0.03125, + "rollout/mismatch_ratio": 0.03125, + "rollout/truncated_ratio": 0.75, + "sampling/importance_sampling_ratio/max": 2.822704792022705, + "sampling/importance_sampling_ratio/mean": 0.4837469756603241, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.2067627906799316, + "sampling/sampling_logp_difference/mean": 0.021238919347524643, + "step": 24, + "step_time": 423.74345705999986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.828125, + "completions/max_length": 1469.0, + "completions/max_terminated_length": 854.0, + "completions/mean_length": 757.296875, + "completions/mean_terminated_length": 445.2727355957031, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "entropy": 0.3146928045898676, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005950183607637882, + "kl": 0.0024588965916336747, + "learning_rate": 3.600985378894086e-06, + "loss": -0.0029, + "num_tokens": 1298023.0, + "reward": 0.12843748927116394, + "reward_std": 0.3103031814098358, + "rewards/alfworld_rollout_reward_func/mean": 0.12843750417232513, + "rewards/alfworld_rollout_reward_func/std": 0.39419689774513245, + "rollout/action_tokens_mean": 757.296875, + "rollout/episode_turns_mean": 27.515625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.828125, + "sampling/importance_sampling_ratio/max": 1.9042965173721313, + "sampling/importance_sampling_ratio/mean": 0.30797556042671204, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.0986688137054443, + "sampling/sampling_logp_difference/mean": 0.0199856199324131, + "step": 25, + "step_time": 379.6568018289963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.796875, + "completions/max_length": 1607.0, + "completions/max_terminated_length": 975.0, + "completions/mean_length": 820.5, + "completions/mean_terminated_length": 396.923095703125, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "entropy": 0.301429130602628, + "epoch": 0.208, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006396389566361904, + "kl": 0.001548040645502624, + "learning_rate": 3.4858896377832966e-06, + "loss": 0.0009, + "num_tokens": 1350535.0, + "reward": 0.1639062464237213, + "reward_std": 0.32607388496398926, + "rewards/alfworld_rollout_reward_func/mean": 0.1639062464237213, + "rewards/alfworld_rollout_reward_func/std": 0.418832927942276, + "rollout/action_tokens_mean": 820.5, + "rollout/episode_turns_mean": 26.59375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.796875, + "sampling/importance_sampling_ratio/max": 2.7253878116607666, + "sampling/importance_sampling_ratio/mean": 0.46614933013916016, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.6588082313537598, + "sampling/sampling_logp_difference/mean": 0.018608851358294487, + "step": 26, + "step_time": 458.5120370709956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.8125, + "completions/max_length": 1363.0, + "completions/max_terminated_length": 851.0, + "completions/mean_length": 843.46875, + "completions/mean_terminated_length": 518.0833740234375, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "entropy": 0.3490098095498979, + "epoch": 0.216, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00987387914210558, + "kl": 0.001589072278875392, + "learning_rate": 3.3682631321120507e-06, + "loss": -0.0109, + "num_tokens": 1404517.0, + "reward": 0.14921875298023224, + "reward_std": 0.3126556873321533, + "rewards/alfworld_rollout_reward_func/mean": 0.14921875298023224, + "rewards/alfworld_rollout_reward_func/std": 0.40770089626312256, + "rollout/action_tokens_mean": 843.46875, + "rollout/episode_turns_mean": 27.703125, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.8125, + "sampling/importance_sampling_ratio/max": 2.323033332824707, + "sampling/importance_sampling_ratio/mean": 0.4162726402282715, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.8558279275894165, + "sampling/sampling_logp_difference/mean": 0.020594514906406403, + "step": 27, + "step_time": 366.20077905700964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 2369.0, + "completions/max_terminated_length": 1151.0, + "completions/mean_length": 727.859375, + "completions/mean_terminated_length": 404.3571472167969, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.30788369150832295, + "epoch": 0.224, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01545465737581253, + "kl": 0.0016130298517964548, + "learning_rate": 3.248407807433396e-06, + "loss": 0.012, + "num_tokens": 1451100.0, + "reward": 0.3981249928474426, + "reward_std": 0.5213440656661987, + "rewards/alfworld_rollout_reward_func/mean": 0.3981249928474426, + "rewards/alfworld_rollout_reward_func/std": 0.5216788649559021, + "rollout/action_tokens_mean": 727.859375, + "rollout/episode_turns_mean": 22.984375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.5625, + "sampling/importance_sampling_ratio/max": 2.7356457710266113, + "sampling/importance_sampling_ratio/mean": 0.6839978694915771, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.65842604637146, + "sampling/sampling_logp_difference/mean": 0.019816581159830093, + "step": 28, + "step_time": 317.7198553449962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 1781.0, + "completions/max_terminated_length": 980.0, + "completions/mean_length": 844.03125, + "completions/mean_terminated_length": 524.75, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "entropy": 0.3514281096868217, + "epoch": 0.232, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010329142212867737, + "kl": 0.0034811663499567658, + "learning_rate": 3.1266313306468018e-06, + "loss": -0.0153, + "num_tokens": 1505118.0, + "reward": 0.20656250417232513, + "reward_std": 0.29942071437835693, + "rewards/alfworld_rollout_reward_func/mean": 0.20656250417232513, + "rewards/alfworld_rollout_reward_func/std": 0.4480122923851013, + "rollout/action_tokens_mean": 844.03125, + "rollout/episode_turns_mean": 26.890625, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.75, + "sampling/importance_sampling_ratio/max": 2.6475582122802734, + "sampling/importance_sampling_ratio/mean": 0.39373910427093506, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 5.319100856781006, + "sampling/sampling_logp_difference/mean": 0.022610869258642197, + "step": 29, + "step_time": 372.01922321400525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 1465.0, + "completions/max_terminated_length": 804.0, + "completions/mean_length": 781.265625, + "completions/mean_terminated_length": 451.8125, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.2927012937143445, + "epoch": 0.24, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006770053878426552, + "kl": 0.0015057786895340541, + "learning_rate": 3.0032463002216504e-06, + "loss": 0.0038, + "num_tokens": 1555119.0, + "reward": 0.21671874821186066, + "reward_std": 0.24240338802337646, + "rewards/alfworld_rollout_reward_func/mean": 0.21671876311302185, + "rewards/alfworld_rollout_reward_func/std": 0.44675213098526, + "rollout/action_tokens_mean": 781.265625, + "rollout/episode_turns_mean": 26.21875, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.75, + "sampling/importance_sampling_ratio/max": 2.8479506969451904, + "sampling/importance_sampling_ratio/mean": 0.49506521224975586, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.7825241088867188, + "sampling/sampling_logp_difference/mean": 0.01828092336654663, + "step": 30, + "step_time": 382.84014304800075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.8125, + "completions/max_length": 1591.0, + "completions/max_terminated_length": 899.0, + "completions/mean_length": 807.0625, + "completions/mean_terminated_length": 503.8333435058594, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "entropy": 0.33275228925049305, + "epoch": 0.248, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009322401136159897, + "kl": 0.0018803979564836482, + "learning_rate": 2.878569443761442e-06, + "loss": -0.0078, + "num_tokens": 1606771.0, + "reward": 0.14656250178813934, + "reward_std": 0.337169349193573, + "rewards/alfworld_rollout_reward_func/mean": 0.14656248688697815, + "rewards/alfworld_rollout_reward_func/std": 0.4100453853607178, + "rollout/action_tokens_mean": 807.0625, + "rollout/episode_turns_mean": 27.546875, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.8125, + "sampling/importance_sampling_ratio/max": 2.9021573066711426, + "sampling/importance_sampling_ratio/mean": 0.42428821325302124, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.855207920074463, + "sampling/sampling_logp_difference/mean": 0.021321609616279602, + "step": 31, + "step_time": 494.6054373920015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.796875, + "completions/max_length": 1383.0, + "completions/max_terminated_length": 1038.0, + "completions/mean_length": 791.734375, + "completions/mean_terminated_length": 412.923095703125, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "entropy": 0.38143961410969496, + "epoch": 0.256, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010985196568071842, + "kl": 0.0017831376208050642, + "learning_rate": 2.752920804968581e-06, + "loss": -0.0093, + "num_tokens": 1657442.0, + "reward": 0.14656248688697815, + "reward_std": 0.27167972922325134, + "rewards/alfworld_rollout_reward_func/mean": 0.14656250178813934, + "rewards/alfworld_rollout_reward_func/std": 0.4263809323310852, + "rollout/action_tokens_mean": 791.734375, + "rollout/episode_turns_mean": 26.5, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.796875, + "sampling/importance_sampling_ratio/max": 2.170595169067383, + "sampling/importance_sampling_ratio/mean": 0.39966434240341187, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.398648977279663, + "sampling/sampling_logp_difference/mean": 0.022872522473335266, + "step": 32, + "step_time": 305.8924537160019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.734375, + "completions/max_length": 1564.0, + "completions/max_terminated_length": 1151.0, + "completions/mean_length": 914.953125, + "completions/mean_terminated_length": 616.4705810546875, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.39743291586637497, + "epoch": 0.264, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010168999433517456, + "kl": 0.0017962153833650518, + "learning_rate": 2.626622922096782e-06, + "loss": -0.0045, + "num_tokens": 1715999.0, + "reward": 0.21843750774860382, + "reward_std": 0.4097728729248047, + "rewards/alfworld_rollout_reward_func/mean": 0.21843750774860382, + "rewards/alfworld_rollout_reward_func/std": 0.46011462807655334, + "rollout/action_tokens_mean": 914.953125, + "rollout/episode_turns_mean": 27.140625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.734375, + "sampling/importance_sampling_ratio/max": 2.6107707023620605, + "sampling/importance_sampling_ratio/mean": 0.386021226644516, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.2772090435028076, + "sampling/sampling_logp_difference/mean": 0.02191011980175972, + "step": 33, + "step_time": 360.4347860389935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.796875, + "completions/max_length": 1527.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 862.9375, + "completions/mean_terminated_length": 506.3077087402344, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.35848523303866386, + "epoch": 0.272, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01134358998388052, + "kl": 0.001760420509526739, + "learning_rate": 2.5e-06, + "loss": 0.016, + "num_tokens": 1771227.0, + "reward": 0.15781250596046448, + "reward_std": 0.29862138628959656, + "rewards/alfworld_rollout_reward_func/mean": 0.15781249105930328, + "rewards/alfworld_rollout_reward_func/std": 0.4202209413051605, + "rollout/action_tokens_mean": 862.9375, + "rollout/episode_turns_mean": 27.515625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.796875, + "sampling/importance_sampling_ratio/max": 2.3946166038513184, + "sampling/importance_sampling_ratio/mean": 0.41097065806388855, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.3659703731536865, + "sampling/sampling_logp_difference/mean": 0.020839743316173553, + "step": 34, + "step_time": 255.86244797499785 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.640625, + "completions/max_length": 1503.0, + "completions/max_terminated_length": 1127.0, + "completions/mean_length": 801.078125, + "completions/mean_terminated_length": 446.0434875488281, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "entropy": 0.33974334690719843, + "epoch": 0.28, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007748698350042105, + "kl": 0.0017098757416533772, + "learning_rate": 2.3733770779032185e-06, + "loss": 0.0012, + "num_tokens": 1822496.0, + "reward": 0.3109375238418579, + "reward_std": 0.34154438972473145, + "rewards/alfworld_rollout_reward_func/mean": 0.3109375238418579, + "rewards/alfworld_rollout_reward_func/std": 0.5012863874435425, + "rollout/action_tokens_mean": 801.078125, + "rollout/episode_turns_mean": 24.203125, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.640625, + "sampling/importance_sampling_ratio/max": 2.6798601150512695, + "sampling/importance_sampling_ratio/mean": 0.4648974537849426, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.4488577842712402, + "sampling/sampling_logp_difference/mean": 0.020977256819605827, + "step": 35, + "step_time": 369.7442883429976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 1817.0, + "completions/max_terminated_length": 1098.0, + "completions/mean_length": 763.078125, + "completions/mean_terminated_length": 383.21429443359375, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "entropy": 0.37786182574927807, + "epoch": 0.288, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012169571593403816, + "kl": 0.0024242461659014225, + "learning_rate": 2.24707919503142e-06, + "loss": 0.0067, + "num_tokens": 1871333.0, + "reward": 0.3814062476158142, + "reward_std": 0.325703501701355, + "rewards/alfworld_rollout_reward_func/mean": 0.3814062476158142, + "rewards/alfworld_rollout_reward_func/std": 0.5243707299232483, + "rollout/action_tokens_mean": 763.078125, + "rollout/episode_turns_mean": 22.328125, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.5625, + "sampling/importance_sampling_ratio/max": 2.630308151245117, + "sampling/importance_sampling_ratio/mean": 0.4907624125480652, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.9331345558166504, + "sampling/sampling_logp_difference/mean": 0.022463176399469376, + "step": 36, + "step_time": 452.8674500970046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.453125, + "completions/max_length": 1560.0, + "completions/max_terminated_length": 1173.0, + "completions/mean_length": 592.9375, + "completions/mean_terminated_length": 322.4285583496094, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "entropy": 0.2960165496915579, + "epoch": 0.296, + "frac_reward_zero_std": 0.125, + "grad_norm": 0.009033096954226494, + "kl": 0.0017318885384156602, + "learning_rate": 2.1214305562385592e-06, + "loss": 0.0028, + "num_tokens": 1909281.0, + "reward": 0.5143750309944153, + "reward_std": 0.3860568404197693, + "rewards/alfworld_rollout_reward_func/mean": 0.5143750309944153, + "rewards/alfworld_rollout_reward_func/std": 0.5290928483009338, + "rollout/action_tokens_mean": 592.9375, + "rollout/episode_turns_mean": 19.015625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.453125, + "sampling/importance_sampling_ratio/max": 2.8567276000976562, + "sampling/importance_sampling_ratio/mean": 0.5531047582626343, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.304647445678711, + "sampling/sampling_logp_difference/mean": 0.01983683556318283, + "step": 37, + "step_time": 388.3879080370025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 1343.0, + "completions/max_terminated_length": 792.0, + "completions/mean_length": 662.140625, + "completions/mean_terminated_length": 343.8333435058594, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.32609579246491194, + "epoch": 0.304, + "frac_reward_zero_std": 0.125, + "grad_norm": 0.014773277565836906, + "kl": 0.0018760439170364407, + "learning_rate": 1.9967536997783495e-06, + "loss": 0.0014, + "num_tokens": 1951658.0, + "reward": 0.4350000023841858, + "reward_std": 0.3387959599494934, + "rewards/alfworld_rollout_reward_func/mean": 0.4350000023841858, + "rewards/alfworld_rollout_reward_func/std": 0.5278016924858093, + "rollout/action_tokens_mean": 662.140625, + "rollout/episode_turns_mean": 21.40625, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.53125, + "sampling/importance_sampling_ratio/max": 2.9260754585266113, + "sampling/importance_sampling_ratio/mean": 0.563366711139679, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.768939971923828, + "sampling/sampling_logp_difference/mean": 0.022423237562179565, + "step": 38, + "step_time": 436.1079989810023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.734375, + "completions/max_length": 1869.0, + "completions/max_terminated_length": 1018.0, + "completions/mean_length": 943.40625, + "completions/mean_terminated_length": 584.4117431640625, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.3758079996332526, + "epoch": 0.312, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01313227228820324, + "kl": 0.0022639515991613735, + "learning_rate": 1.8733686693531986e-06, + "loss": -0.0025, + "num_tokens": 2012036.0, + "reward": 0.21578124165534973, + "reward_std": 0.4323212504386902, + "rewards/alfworld_rollout_reward_func/mean": 0.21578124165534973, + "rewards/alfworld_rollout_reward_func/std": 0.46428921818733215, + "rollout/action_tokens_mean": 943.40625, + "rollout/episode_turns_mean": 26.90625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.734375, + "sampling/importance_sampling_ratio/max": 2.5490753650665283, + "sampling/importance_sampling_ratio/mean": 0.3670840859413147, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.389880657196045, + "sampling/sampling_logp_difference/mean": 0.021743088960647583, + "step": 39, + "step_time": 354.37862886100265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.671875, + "completions/max_length": 1334.0, + "completions/max_terminated_length": 1062.0, + "completions/mean_length": 701.15625, + "completions/mean_terminated_length": 308.1428527832031, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "entropy": 0.2994346688501537, + "epoch": 0.32, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009722444228827953, + "kl": 0.0016241020402958384, + "learning_rate": 1.7515921925666053e-06, + "loss": -0.0206, + "num_tokens": 2056910.0, + "reward": 0.29640626907348633, + "reward_std": 0.23870769143104553, + "rewards/alfworld_rollout_reward_func/mean": 0.29640626907348633, + "rewards/alfworld_rollout_reward_func/std": 0.4918813109397888, + "rollout/action_tokens_mean": 701.15625, + "rollout/episode_turns_mean": 23.625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.671875, + "sampling/importance_sampling_ratio/max": 2.825843572616577, + "sampling/importance_sampling_ratio/mean": 0.5806776881217957, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.3793482780456543, + "sampling/sampling_logp_difference/mean": 0.01931983232498169, + "step": 40, + "step_time": 355.1650720970065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.71875, + "completions/max_length": 1877.0, + "completions/max_terminated_length": 997.0, + "completions/mean_length": 860.609375, + "completions/mean_terminated_length": 526.0, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.33433423191308975, + "epoch": 0.328, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01665978506207466, + "kl": 0.001694458733254578, + "learning_rate": 1.6317368678879497e-06, + "loss": 0.0013, + "num_tokens": 2111989.0, + "reward": 0.23156249523162842, + "reward_std": 0.42284566164016724, + "rewards/alfworld_rollout_reward_func/mean": 0.23156249523162842, + "rewards/alfworld_rollout_reward_func/std": 0.4703366458415985, + "rollout/action_tokens_mean": 860.609375, + "rollout/episode_turns_mean": 26.859375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.71875, + "sampling/importance_sampling_ratio/max": 2.746248960494995, + "sampling/importance_sampling_ratio/mean": 0.47027722001075745, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.0510313510894775, + "sampling/sampling_logp_difference/mean": 0.019619014114141464, + "step": 41, + "step_time": 383.9065069159915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.578125, + "completions/max_length": 1491.0, + "completions/max_terminated_length": 1206.0, + "completions/mean_length": 699.109375, + "completions/mean_terminated_length": 410.9259338378906, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "entropy": 0.3300422574393451, + "epoch": 0.336, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011248192749917507, + "kl": 0.0020856883202213794, + "learning_rate": 1.5141103622167042e-06, + "loss": 0.0075, + "num_tokens": 2156732.0, + "reward": 0.37812501192092896, + "reward_std": 0.3792918920516968, + "rewards/alfworld_rollout_reward_func/mean": 0.37812501192092896, + "rewards/alfworld_rollout_reward_func/std": 0.5151386857032776, + "rollout/action_tokens_mean": 699.109375, + "rollout/episode_turns_mean": 23.015625, + "rollout/mismatch_mean": 0.03125, + "rollout/mismatch_ratio": 0.03125, + "rollout/truncated_ratio": 0.578125, + "sampling/importance_sampling_ratio/max": 2.8479621410369873, + "sampling/importance_sampling_ratio/mean": 0.49075883626937866, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.1700198650360107, + "sampling/sampling_logp_difference/mean": 0.02142202854156494, + "step": 42, + "step_time": 342.9056702470225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.71875, + "completions/max_length": 1746.0, + "completions/max_terminated_length": 864.0, + "completions/mean_length": 780.5, + "completions/mean_terminated_length": 496.6111145019531, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "entropy": 0.31365567492321134, + "epoch": 0.344, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013282958418130875, + "kl": 0.0016229442680923967, + "learning_rate": 1.3990146211059141e-06, + "loss": -0.0022, + "num_tokens": 2206684.0, + "reward": 0.2423437535762787, + "reward_std": 0.44438114762306213, + "rewards/alfworld_rollout_reward_func/mean": 0.2423437535762787, + "rewards/alfworld_rollout_reward_func/std": 0.47146087884902954, + "rollout/action_tokens_mean": 780.5, + "rollout/episode_turns_mean": 26.484375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.71875, + "sampling/importance_sampling_ratio/max": 2.7073540687561035, + "sampling/importance_sampling_ratio/mean": 0.548378586769104, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.89083194732666, + "sampling/sampling_logp_difference/mean": 0.020154234021902084, + "step": 43, + "step_time": 465.0683610840024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.765625, + "completions/max_length": 2028.0, + "completions/max_terminated_length": 1179.0, + "completions/mean_length": 882.0625, + "completions/mean_terminated_length": 563.0, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "entropy": 0.30738420970737934, + "epoch": 0.352, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011168000288307667, + "kl": 0.001302105893046246, + "learning_rate": 1.286745093672298e-06, + "loss": 0.0006, + "num_tokens": 2263136.0, + "reward": 0.18671876192092896, + "reward_std": 0.23547054827213287, + "rewards/alfworld_rollout_reward_func/mean": 0.18671876192092896, + "rewards/alfworld_rollout_reward_func/std": 0.4461868703365326, + "rollout/action_tokens_mean": 882.0625, + "rollout/episode_turns_mean": 26.578125, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.765625, + "sampling/importance_sampling_ratio/max": 2.5664165019989014, + "sampling/importance_sampling_ratio/mean": 0.4952065646648407, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 8.617661476135254, + "sampling/sampling_logp_difference/mean": 0.019173569977283478, + "step": 44, + "step_time": 361.35297568800524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 1390.0, + "completions/max_terminated_length": 1011.0, + "completions/mean_length": 807.96875, + "completions/mean_terminated_length": 565.9091186523438, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.33469977881759405, + "epoch": 0.36, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005941484589129686, + "kl": 0.001596126568983891, + "learning_rate": 1.1775899741825947e-06, + "loss": 0.0031, + "num_tokens": 2314846.0, + "reward": 0.2996875047683716, + "reward_std": 0.383698046207428, + "rewards/alfworld_rollout_reward_func/mean": 0.2996875047683716, + "rewards/alfworld_rollout_reward_func/std": 0.48867806792259216, + "rollout/action_tokens_mean": 807.96875, + "rollout/episode_turns_mean": 25.65625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.65625, + "sampling/importance_sampling_ratio/max": 1.910278081893921, + "sampling/importance_sampling_ratio/mean": 0.4718725383281708, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.8846027851104736, + "sampling/sampling_logp_difference/mean": 0.021218668669462204, + "step": 45, + "step_time": 477.9487361019965 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.609375, + "completions/max_length": 1381.0, + "completions/max_terminated_length": 827.0, + "completions/mean_length": 659.234375, + "completions/mean_terminated_length": 271.7200012207031, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.3191372868604958, + "epoch": 0.368, + "frac_reward_zero_std": 0.125, + "grad_norm": 0.01500615943223238, + "kl": 0.0017743182979756966, + "learning_rate": 1.0718294622630188e-06, + "loss": -0.007, + "num_tokens": 2357037.0, + "reward": 0.3592187464237213, + "reward_std": 0.3250337839126587, + "rewards/alfworld_rollout_reward_func/mean": 0.3592187464237213, + "rewards/alfworld_rollout_reward_func/std": 0.5135136246681213, + "rollout/action_tokens_mean": 659.234375, + "rollout/episode_turns_mean": 21.640625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.609375, + "sampling/importance_sampling_ratio/max": 2.703761100769043, + "sampling/importance_sampling_ratio/mean": 0.6121923923492432, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.5571508407592773, + "sampling/sampling_logp_difference/mean": 0.02017129212617874, + "step": 46, + "step_time": 368.9170330479901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 1449.0, + "completions/max_terminated_length": 879.0, + "completions/mean_length": 630.171875, + "completions/mean_terminated_length": 346.20587158203125, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "entropy": 0.33235412929207087, + "epoch": 0.376, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014015787281095982, + "kl": 0.001809486606362043, + "learning_rate": 9.697350436308428e-07, + "loss": 0.0171, + "num_tokens": 2397368.0, + "reward": 0.4931250214576721, + "reward_std": 0.29200682044029236, + "rewards/alfworld_rollout_reward_func/mean": 0.4931250214576721, + "rewards/alfworld_rollout_reward_func/std": 0.5321918725967407, + "rollout/action_tokens_mean": 630.171875, + "rollout/episode_turns_mean": 19.6875, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.46875, + "sampling/importance_sampling_ratio/max": 2.7606446743011475, + "sampling/importance_sampling_ratio/mean": 0.6627721190452576, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.4474594593048096, + "sampling/sampling_logp_difference/mean": 0.02060038223862648, + "step": 47, + "step_time": 259.2473690390034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.875, + "completions/max_length": 2126.0, + "completions/max_terminated_length": 1218.0, + "completions/mean_length": 943.59375, + "completions/mean_terminated_length": 424.0, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.3577259639278054, + "epoch": 0.384, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014235694892704487, + "kl": 0.0015301553139579482, + "learning_rate": 8.71568793194445e-07, + "loss": 0.0326, + "num_tokens": 2457758.0, + "reward": 0.06859375536441803, + "reward_std": 0.24723710119724274, + "rewards/alfworld_rollout_reward_func/mean": 0.06859375536441803, + "rewards/alfworld_rollout_reward_func/std": 0.3545883595943451, + "rollout/action_tokens_mean": 943.59375, + "rollout/episode_turns_mean": 27.734375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.875, + "sampling/importance_sampling_ratio/max": 2.791482925415039, + "sampling/importance_sampling_ratio/mean": 0.5324603319168091, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.8059403896331787, + "sampling/sampling_logp_difference/mean": 0.020999200642108917, + "step": 48, + "step_time": 384.36283810400346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.734375, + "completions/max_length": 1635.0, + "completions/max_terminated_length": 980.0, + "completions/mean_length": 861.203125, + "completions/mean_terminated_length": 481.4705810546875, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "entropy": 0.36975698871538043, + "epoch": 0.392, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006656537763774395, + "kl": 0.0018834842448995914, + "learning_rate": 7.775827023107835e-07, + "loss": 0.0044, + "num_tokens": 2512875.0, + "reward": 0.21015624701976776, + "reward_std": 0.3197978734970093, + "rewards/alfworld_rollout_reward_func/mean": 0.21015626192092896, + "rewards/alfworld_rollout_reward_func/std": 0.4713321030139923, + "rollout/action_tokens_mean": 861.203125, + "rollout/episode_turns_mean": 26.59375, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.734375, + "sampling/importance_sampling_ratio/max": 2.697384834289551, + "sampling/importance_sampling_ratio/mean": 0.4099380373954773, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 4.545639991760254, + "sampling/sampling_logp_difference/mean": 0.02258184552192688, + "step": 49, + "step_time": 397.8871677600073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 1624.0, + "completions/max_terminated_length": 910.0, + "completions/mean_length": 887.515625, + "completions/mean_terminated_length": 580.0625, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.3667739760130644, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013544962741434574, + "kl": 0.0015946346466080286, + "learning_rate": 6.880180319272006e-07, + "loss": 0.0071, + "num_tokens": 2569676.0, + "reward": 0.1901562511920929, + "reward_std": 0.45852425694465637, + "rewards/alfworld_rollout_reward_func/mean": 0.1901562511920929, + "rewards/alfworld_rollout_reward_func/std": 0.4570210576057434, + "rollout/action_tokens_mean": 887.515625, + "rollout/episode_turns_mean": 26.828125, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.75, + "sampling/importance_sampling_ratio/max": 2.99810791015625, + "sampling/importance_sampling_ratio/mean": 0.5466522574424744, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 4.044807434082031, + "sampling/sampling_logp_difference/mean": 0.02207544818520546, + "step": 50, + "step_time": 363.5887358939908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 1596.0, + "completions/max_terminated_length": 1015.0, + "completions/mean_length": 773.125, + "completions/mean_terminated_length": 392.4090881347656, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "entropy": 0.3592249136418104, + "epoch": 0.408, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01083005964756012, + "kl": 0.002072367888104054, + "learning_rate": 6.031046932680229e-07, + "loss": 0.0148, + "num_tokens": 2619156.0, + "reward": 0.30000001192092896, + "reward_std": 0.43087929487228394, + "rewards/alfworld_rollout_reward_func/mean": 0.30000001192092896, + "rewards/alfworld_rollout_reward_func/std": 0.500076174736023, + "rollout/action_tokens_mean": 773.125, + "rollout/episode_turns_mean": 24.046875, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.65625, + "sampling/importance_sampling_ratio/max": 2.189059257507324, + "sampling/importance_sampling_ratio/mean": 0.4208452105522156, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.6584203243255615, + "sampling/sampling_logp_difference/mean": 0.022034121677279472, + "step": 51, + "step_time": 381.66528784099137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 1738.0, + "completions/max_terminated_length": 1079.0, + "completions/mean_length": 780.3125, + "completions/mean_terminated_length": 445.5769348144531, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "entropy": 0.3415462737902999, + "epoch": 0.416, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005050386302173138, + "kl": 0.0017878458802442765, + "learning_rate": 5.23060657655754e-07, + "loss": -0.0079, + "num_tokens": 2669096.0, + "reward": 0.3726562559604645, + "reward_std": 0.36754581332206726, + "rewards/alfworld_rollout_reward_func/mean": 0.3726562559604645, + "rewards/alfworld_rollout_reward_func/std": 0.5065045356750488, + "rollout/action_tokens_mean": 780.3125, + "rollout/episode_turns_mean": 23.90625, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.59375, + "sampling/importance_sampling_ratio/max": 2.1458444595336914, + "sampling/importance_sampling_ratio/mean": 0.4066748023033142, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.5774283409118652, + "sampling/sampling_logp_difference/mean": 0.022033972665667534, + "step": 52, + "step_time": 482.9236067970087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.734375, + "completions/max_length": 1387.0, + "completions/max_terminated_length": 1161.0, + "completions/mean_length": 819.3125, + "completions/mean_terminated_length": 579.8823852539062, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "entropy": 0.33714828826487064, + "epoch": 0.424, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010500618256628513, + "kl": 0.0019351580704096705, + "learning_rate": 4.480913969818099e-07, + "loss": -0.0017, + "num_tokens": 2721532.0, + "reward": 0.21718750894069672, + "reward_std": 0.38691049814224243, + "rewards/alfworld_rollout_reward_func/mean": 0.21718750894069672, + "rewards/alfworld_rollout_reward_func/std": 0.4590102434158325, + "rollout/action_tokens_mean": 819.3125, + "rollout/episode_turns_mean": 27.015625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.734375, + "sampling/importance_sampling_ratio/max": 2.4449539184570312, + "sampling/importance_sampling_ratio/mean": 0.38772356510162354, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.6585437059402466, + "sampling/sampling_logp_difference/mean": 0.020567741245031357, + "step": 53, + "step_time": 440.7185651119944 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.765625, + "completions/max_length": 1538.0, + "completions/max_terminated_length": 1077.0, + "completions/mean_length": 858.40625, + "completions/mean_terminated_length": 563.4666748046875, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "entropy": 0.3832448711618781, + "epoch": 0.432, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00624613743275404, + "kl": 0.0019235094769101124, + "learning_rate": 3.7838935626312246e-07, + "loss": -0.003, + "num_tokens": 2776470.0, + "reward": 0.18359375, + "reward_std": 0.32859885692596436, + "rewards/alfworld_rollout_reward_func/mean": 0.18359375, + "rewards/alfworld_rollout_reward_func/std": 0.44527989625930786, + "rollout/action_tokens_mean": 858.40625, + "rollout/episode_turns_mean": 26.546875, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.765625, + "sampling/importance_sampling_ratio/max": 2.0156307220458984, + "sampling/importance_sampling_ratio/mean": 0.2892346978187561, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.6584253311157227, + "sampling/sampling_logp_difference/mean": 0.022003933787345886, + "step": 54, + "step_time": 433.99726870600716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.640625, + "completions/max_length": 1862.0, + "completions/max_terminated_length": 1001.0, + "completions/mean_length": 774.25, + "completions/mean_terminated_length": 319.9565124511719, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.3484980883076787, + "epoch": 0.44, + "frac_reward_zero_std": 0.125, + "grad_norm": 0.019602373242378235, + "kl": 0.0015943678226904012, + "learning_rate": 3.141334596385448e-07, + "loss": 0.0162, + "num_tokens": 2826022.0, + "reward": 0.30687499046325684, + "reward_std": 0.2975793182849884, + "rewards/alfworld_rollout_reward_func/mean": 0.30687499046325684, + "rewards/alfworld_rollout_reward_func/std": 0.5158684253692627, + "rollout/action_tokens_mean": 774.25, + "rollout/episode_turns_mean": 22.890625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.640625, + "sampling/importance_sampling_ratio/max": 2.4803109169006348, + "sampling/importance_sampling_ratio/mean": 0.5007343292236328, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.6167523860931396, + "sampling/sampling_logp_difference/mean": 0.021400175988674164, + "step": 55, + "step_time": 371.2160142980065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.734375, + "completions/max_length": 1620.0, + "completions/max_terminated_length": 957.0, + "completions/mean_length": 795.484375, + "completions/mean_terminated_length": 376.058837890625, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "entropy": 0.32310054916888475, + "epoch": 0.448, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01010950654745102, + "kl": 0.0015955448616296053, + "learning_rate": 2.5548865107314606e-07, + "loss": 0.0093, + "num_tokens": 2876933.0, + "reward": 0.22109374403953552, + "reward_std": 0.31529495120048523, + "rewards/alfworld_rollout_reward_func/mean": 0.22109374403953552, + "rewards/alfworld_rollout_reward_func/std": 0.4678628444671631, + "rollout/action_tokens_mean": 795.484375, + "rollout/episode_turns_mean": 25.328125, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.734375, + "sampling/importance_sampling_ratio/max": 2.8859243392944336, + "sampling/importance_sampling_ratio/mean": 0.5188072919845581, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 5.368804454803467, + "sampling/sampling_logp_difference/mean": 0.020041368901729584, + "step": 56, + "step_time": 424.94623591700656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.71875, + "completions/max_length": 1384.0, + "completions/max_terminated_length": 1021.0, + "completions/mean_length": 818.9375, + "completions/mean_terminated_length": 486.0, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.35508036240935326, + "epoch": 0.456, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010526562109589577, + "kl": 0.0017983552861551289, + "learning_rate": 2.026054709494235e-07, + "loss": -0.0097, + "num_tokens": 2929345.0, + "reward": 0.23046875, + "reward_std": 0.4045883119106293, + "rewards/alfworld_rollout_reward_func/mean": 0.23046875, + "rewards/alfworld_rollout_reward_func/std": 0.475816547870636, + "rollout/action_tokens_mean": 818.9375, + "rollout/episode_turns_mean": 25.953125, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.71875, + "sampling/importance_sampling_ratio/max": 2.9720585346221924, + "sampling/importance_sampling_ratio/mean": 0.5196441411972046, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.8799049854278564, + "sampling/sampling_logp_difference/mean": 0.022473536431789398, + "step": 57, + "step_time": 439.1710239440072 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 1419.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 738.4375, + "completions/mean_terminated_length": 387.2500305175781, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.3470684285275638, + "epoch": 0.464, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00702791940420866, + "kl": 0.0016746247110859258, + "learning_rate": 1.5561966963229925e-07, + "loss": -0.0058, + "num_tokens": 2976605.0, + "reward": 0.3982812762260437, + "reward_std": 0.37868061661720276, + "rewards/alfworld_rollout_reward_func/mean": 0.3982812464237213, + "rewards/alfworld_rollout_reward_func/std": 0.5275837779045105, + "rollout/action_tokens_mean": 738.4375, + "rollout/episode_turns_mean": 22.078125, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.5625, + "sampling/importance_sampling_ratio/max": 1.7666839361190796, + "sampling/importance_sampling_ratio/mean": 0.42546379566192627, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.1239027976989746, + "sampling/sampling_logp_difference/mean": 0.02082432433962822, + "step": 58, + "step_time": 360.1903216189894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 1488.0, + "completions/max_terminated_length": 1112.0, + "completions/mean_length": 657.859375, + "completions/mean_terminated_length": 380.02777099609375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.3138961121439934, + "epoch": 0.472, + "frac_reward_zero_std": 0.125, + "grad_norm": 0.016277866438031197, + "kl": 0.0016064580941019813, + "learning_rate": 1.1465185899987797e-07, + "loss": 0.0023, + "num_tokens": 3018708.0, + "reward": 0.5278124809265137, + "reward_std": 0.386292964220047, + "rewards/alfworld_rollout_reward_func/mean": 0.5278124809265137, + "rewards/alfworld_rollout_reward_func/std": 0.5249382853507996, + "rollout/action_tokens_mean": 657.859375, + "rollout/episode_turns_mean": 19.984375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.4375, + "sampling/importance_sampling_ratio/max": 2.904142379760742, + "sampling/importance_sampling_ratio/mean": 0.5997130274772644, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.4242944717407227, + "sampling/sampling_logp_difference/mean": 0.01990351639688015, + "step": 59, + "step_time": 324.923169317015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 1532.0, + "completions/max_terminated_length": 1118.0, + "completions/mean_length": 825.765625, + "completions/mean_terminated_length": 551.6818237304688, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.3388841552659869, + "epoch": 0.48, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01379719190299511, + "kl": 0.001881336524093058, + "learning_rate": 7.980720283448957e-08, + "loss": 0.0022, + "num_tokens": 3071557.0, + "reward": 0.28171876072883606, + "reward_std": 0.49070847034454346, + "rewards/alfworld_rollout_reward_func/mean": 0.28171873092651367, + "rewards/alfworld_rollout_reward_func/std": 0.50815749168396, + "rollout/action_tokens_mean": 825.765625, + "rollout/episode_turns_mean": 25.703125, + "rollout/mismatch_mean": 0.015625, + "rollout/mismatch_ratio": 0.015625, + "rollout/truncated_ratio": 0.65625, + "sampling/importance_sampling_ratio/max": 2.6620919704437256, + "sampling/importance_sampling_ratio/mean": 0.47809475660324097, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 4.668699741363525, + "sampling/sampling_logp_difference/mean": 0.02150684781372547, + "step": 60, + "step_time": 377.0457431760042 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.703125, + "completions/max_length": 1838.0, + "completions/max_terminated_length": 806.0, + "completions/mean_length": 792.515625, + "completions/mean_terminated_length": 347.631591796875, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "entropy": 0.36354396492242813, + "epoch": 0.488, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005873848684132099, + "kl": 0.002195166100136703, + "learning_rate": 5.117514686876379e-08, + "loss": 0.0038, + "num_tokens": 3122278.0, + "reward": 0.2498437464237213, + "reward_std": 0.3333018124103546, + "rewards/alfworld_rollout_reward_func/mean": 0.2498437464237213, + "rewards/alfworld_rollout_reward_func/std": 0.4883873760700226, + "rollout/action_tokens_mean": 792.515625, + "rollout/episode_turns_mean": 24.234375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.703125, + "sampling/importance_sampling_ratio/max": 1.6227197647094727, + "sampling/importance_sampling_ratio/mean": 0.33059564232826233, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 4.1571760177612305, + "sampling/sampling_logp_difference/mean": 0.02179795503616333, + "step": 61, + "step_time": 399.4235413080096 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 1345.0, + "completions/max_terminated_length": 1183.0, + "completions/mean_length": 772.140625, + "completions/mean_terminated_length": 398.20001220703125, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "entropy": 0.3363960413262248, + "epoch": 0.496, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014743300154805183, + "kl": 0.0017252494653803296, + "learning_rate": 2.8829189179721552e-08, + "loss": 0.0094, + "num_tokens": 3171695.0, + "reward": 0.2721875011920929, + "reward_std": 0.40832602977752686, + "rewards/alfworld_rollout_reward_func/mean": 0.2721875011920929, + "rewards/alfworld_rollout_reward_func/std": 0.4889914393424988, + "rollout/action_tokens_mean": 772.140625, + "rollout/episode_turns_mean": 24.515625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.6875, + "sampling/importance_sampling_ratio/max": 2.4185616970062256, + "sampling/importance_sampling_ratio/mean": 0.5585420727729797, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 3.382229804992676, + "sampling/sampling_logp_difference/mean": 0.020675554871559143, + "step": 62, + "step_time": 366.09222524699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.640625, + "completions/max_length": 1560.0, + "completions/max_terminated_length": 1246.0, + "completions/mean_length": 804.015625, + "completions/mean_terminated_length": 414.3913269042969, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3338092272169888, + "epoch": 0.504, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009038163349032402, + "kl": 0.0018702544839470647, + "learning_rate": 1.2826691520262114e-08, + "loss": 0.0236, + "num_tokens": 3223152.0, + "reward": 0.30265626311302185, + "reward_std": 0.3771285116672516, + "rewards/alfworld_rollout_reward_func/mean": 0.30265623331069946, + "rewards/alfworld_rollout_reward_func/std": 0.5124087333679199, + "rollout/action_tokens_mean": 804.015625, + "rollout/episode_turns_mean": 23.90625, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.640625, + "sampling/importance_sampling_ratio/max": 2.994922161102295, + "sampling/importance_sampling_ratio/mean": 0.5557758808135986, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 2.050037384033203, + "sampling/sampling_logp_difference/mean": 0.02032957598567009, + "step": 63, + "step_time": 381.51745763700455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.75, + "completions/max_length": 1427.0, + "completions/max_terminated_length": 964.0, + "completions/mean_length": 838.5, + "completions/mean_terminated_length": 443.9375, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "entropy": 0.3799225674010813, + "epoch": 0.512, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011489775963127613, + "kl": 0.0017125457925430965, + "learning_rate": 3.208732072368104e-09, + "loss": 0.0198, + "num_tokens": 3276816.0, + "reward": 0.20734374225139618, + "reward_std": 0.32793596386909485, + "rewards/alfworld_rollout_reward_func/mean": 0.20734375715255737, + "rewards/alfworld_rollout_reward_func/std": 0.4573083817958832, + "rollout/action_tokens_mean": 838.5, + "rollout/episode_turns_mean": 26.109375, + "rollout/mismatch_mean": 0.0, + "rollout/mismatch_ratio": 0.0, + "rollout/truncated_ratio": 0.75, + "sampling/importance_sampling_ratio/max": 2.7888708114624023, + "sampling/importance_sampling_ratio/mean": 0.423524409532547, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.943666696548462, + "sampling/sampling_logp_difference/mean": 0.02185032144188881, + "step": 64, + "step_time": 388.11920543999804 + } + ], + "logging_steps": 1, + "max_steps": 64, + "num_input_tokens_seen": 3276816, + "num_train_epochs": 1, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}