| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.32, | |
| "eval_steps": 500, | |
| "global_step": 40, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6875, | |
| "completions/max_length": 1707.0, | |
| "completions/max_terminated_length": 915.0, | |
| "completions/mean_length": 812.703125, | |
| "completions/mean_terminated_length": 386.1499938964844, | |
| "completions/min_length": 88.0, | |
| "completions/min_terminated_length": 88.0, | |
| "entropy": 0.319809939712286, | |
| "epoch": 0.008, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.013934657908976078, | |
| "kl": 0.0, | |
| "learning_rate": 0.0, | |
| "loss": 0.0003, | |
| "num_tokens": 52013.0, | |
| "reward": 0.26906251907348633, | |
| "reward_std": 0.43214160203933716, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.26906251907348633, | |
| "rewards/alfworld_rollout_reward_func/std": 0.48894354701042175, | |
| "rollout/action_tokens_mean": 812.703125, | |
| "rollout/episode_turns_mean": 25.0625, | |
| "rollout/mismatch_mean": 0.015625, | |
| "rollout/mismatch_ratio": 0.015625, | |
| "rollout/truncated_ratio": 0.6875, | |
| "sampling/importance_sampling_ratio/max": 2.401656150817871, | |
| "sampling/importance_sampling_ratio/mean": 0.5413336753845215, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.9913136959075928, | |
| "sampling/sampling_logp_difference/mean": 0.020846465602517128, | |
| "step": 1, | |
| "step_time": 340.2447626780008 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.765625, | |
| "completions/max_length": 1338.0, | |
| "completions/max_terminated_length": 912.0, | |
| "completions/mean_length": 822.90625, | |
| "completions/mean_terminated_length": 481.0000305175781, | |
| "completions/min_length": 240.0, | |
| "completions/min_terminated_length": 240.0, | |
| "entropy": 0.33951753890141845, | |
| "epoch": 0.016, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.005984712392091751, | |
| "kl": 0.0, | |
| "learning_rate": 2.5e-06, | |
| "loss": 0.0064, | |
| "num_tokens": 104679.0, | |
| "reward": 0.1837500035762787, | |
| "reward_std": 0.37211528420448303, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.1837500035762787, | |
| "rewards/alfworld_rollout_reward_func/std": 0.44909608364105225, | |
| "rollout/action_tokens_mean": 822.90625, | |
| "rollout/episode_turns_mean": 26.75, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.765625, | |
| "sampling/importance_sampling_ratio/max": 2.0904664993286133, | |
| "sampling/importance_sampling_ratio/mean": 0.30532318353652954, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.2203328609466553, | |
| "sampling/sampling_logp_difference/mean": 0.021448608487844467, | |
| "step": 2, | |
| "step_time": 376.3160819019986 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.75, | |
| "completions/max_length": 1253.0, | |
| "completions/max_terminated_length": 680.0, | |
| "completions/mean_length": 735.65625, | |
| "completions/mean_terminated_length": 380.5, | |
| "completions/min_length": 145.0, | |
| "completions/min_terminated_length": 145.0, | |
| "entropy": 0.32836992759257555, | |
| "epoch": 0.024, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.0066122072748839855, | |
| "kl": 0.001546632844110718, | |
| "learning_rate": 5e-06, | |
| "loss": 0.0026, | |
| "num_tokens": 151761.0, | |
| "reward": 0.21156249940395355, | |
| "reward_std": 0.2755644917488098, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.21156249940395355, | |
| "rewards/alfworld_rollout_reward_func/std": 0.44746631383895874, | |
| "rollout/action_tokens_mean": 735.65625, | |
| "rollout/episode_turns_mean": 26.03125, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.75, | |
| "sampling/importance_sampling_ratio/max": 2.4706838130950928, | |
| "sampling/importance_sampling_ratio/mean": 0.5484369993209839, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.5531697273254395, | |
| "sampling/sampling_logp_difference/mean": 0.020412152633070946, | |
| "step": 3, | |
| "step_time": 359.0840177950013 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.546875, | |
| "completions/max_length": 1880.0, | |
| "completions/max_terminated_length": 807.0, | |
| "completions/mean_length": 721.78125, | |
| "completions/mean_terminated_length": 425.72412109375, | |
| "completions/min_length": 112.0, | |
| "completions/min_terminated_length": 112.0, | |
| "entropy": 0.3564013452269137, | |
| "epoch": 0.032, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.00901270005851984, | |
| "kl": 0.001597001251866459, | |
| "learning_rate": 4.996791267927632e-06, | |
| "loss": 0.0123, | |
| "num_tokens": 197955.0, | |
| "reward": 0.40953126549720764, | |
| "reward_std": 0.48288512229919434, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.40953126549720764, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5298889875411987, | |
| "rollout/action_tokens_mean": 721.78125, | |
| "rollout/episode_turns_mean": 22.859375, | |
| "rollout/mismatch_mean": 0.015625, | |
| "rollout/mismatch_ratio": 0.015625, | |
| "rollout/truncated_ratio": 0.546875, | |
| "sampling/importance_sampling_ratio/max": 2.607266664505005, | |
| "sampling/importance_sampling_ratio/mean": 0.5041748881340027, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.2973906993865967, | |
| "sampling/sampling_logp_difference/mean": 0.022099550813436508, | |
| "step": 4, | |
| "step_time": 324.93751245399835 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6875, | |
| "completions/max_length": 1772.0, | |
| "completions/max_terminated_length": 1135.0, | |
| "completions/mean_length": 823.984375, | |
| "completions/mean_terminated_length": 511.25, | |
| "completions/min_length": 238.0, | |
| "completions/min_terminated_length": 238.0, | |
| "entropy": 0.37002173252403736, | |
| "epoch": 0.04, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.006255371496081352, | |
| "kl": 0.0018659899324120488, | |
| "learning_rate": 4.987173308479738e-06, | |
| "loss": -0.0049, | |
| "num_tokens": 250690.0, | |
| "reward": 0.2629687488079071, | |
| "reward_std": 0.3148724138736725, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2629687786102295, | |
| "rewards/alfworld_rollout_reward_func/std": 0.48142871260643005, | |
| "rollout/action_tokens_mean": 823.984375, | |
| "rollout/episode_turns_mean": 25.65625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.6875, | |
| "sampling/importance_sampling_ratio/max": 2.4376842975616455, | |
| "sampling/importance_sampling_ratio/mean": 0.3966376781463623, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.3790063858032227, | |
| "sampling/sampling_logp_difference/mean": 0.022958241403102875, | |
| "step": 5, | |
| "step_time": 428.7077085350011 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.546875, | |
| "completions/max_length": 1754.0, | |
| "completions/max_terminated_length": 1081.0, | |
| "completions/mean_length": 778.265625, | |
| "completions/mean_terminated_length": 513.0689697265625, | |
| "completions/min_length": 89.0, | |
| "completions/min_terminated_length": 89.0, | |
| "entropy": 0.33683515526354313, | |
| "epoch": 0.048, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.014856722205877304, | |
| "kl": 0.0018188257581641665, | |
| "learning_rate": 4.971170810820279e-06, | |
| "loss": 0.0284, | |
| "num_tokens": 300499.0, | |
| "reward": 0.4112499952316284, | |
| "reward_std": 0.4292137622833252, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.4112499952316284, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5246087312698364, | |
| "rollout/action_tokens_mean": 778.265625, | |
| "rollout/episode_turns_mean": 24.0, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.546875, | |
| "sampling/importance_sampling_ratio/max": 2.469376802444458, | |
| "sampling/importance_sampling_ratio/mean": 0.41494274139404297, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.5756630897521973, | |
| "sampling/sampling_logp_difference/mean": 0.020635541528463364, | |
| "step": 6, | |
| "step_time": 329.3498830619992 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.765625, | |
| "completions/max_length": 1750.0, | |
| "completions/max_terminated_length": 882.0, | |
| "completions/mean_length": 834.359375, | |
| "completions/mean_terminated_length": 538.4000244140625, | |
| "completions/min_length": 142.0, | |
| "completions/min_terminated_length": 142.0, | |
| "entropy": 0.37804630119353533, | |
| "epoch": 0.056, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.011092925444245338, | |
| "kl": 0.0017349459521938115, | |
| "learning_rate": 4.948824853131237e-06, | |
| "loss": 0.0135, | |
| "num_tokens": 353898.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.41170334815979004, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.1875, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4403570294380188, | |
| "rollout/action_tokens_mean": 834.359375, | |
| "rollout/episode_turns_mean": 27.0625, | |
| "rollout/mismatch_mean": 0.015625, | |
| "rollout/mismatch_ratio": 0.015625, | |
| "rollout/truncated_ratio": 0.765625, | |
| "sampling/importance_sampling_ratio/max": 2.2279603481292725, | |
| "sampling/importance_sampling_ratio/mean": 0.46820664405822754, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.733177900314331, | |
| "sampling/sampling_logp_difference/mean": 0.022584954276680946, | |
| "step": 7, | |
| "step_time": 372.10799923799914 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.765625, | |
| "completions/max_length": 1682.0, | |
| "completions/max_terminated_length": 697.0, | |
| "completions/mean_length": 786.5625, | |
| "completions/mean_terminated_length": 380.8000183105469, | |
| "completions/min_length": 180.0, | |
| "completions/min_terminated_length": 180.0, | |
| "entropy": 0.32334376219660044, | |
| "epoch": 0.064, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010468119755387306, | |
| "kl": 0.0017176041819766397, | |
| "learning_rate": 4.920192797165511e-06, | |
| "loss": -0.0059, | |
| "num_tokens": 404238.0, | |
| "reward": 0.1939062476158142, | |
| "reward_std": 0.31658345460891724, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.1939062476158142, | |
| "rewards/alfworld_rollout_reward_func/std": 0.44521304965019226, | |
| "rollout/action_tokens_mean": 786.5625, | |
| "rollout/episode_turns_mean": 25.984375, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.765625, | |
| "sampling/importance_sampling_ratio/max": 2.8437609672546387, | |
| "sampling/importance_sampling_ratio/mean": 0.6038376092910767, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.8486413955688477, | |
| "sampling/sampling_logp_difference/mean": 0.0203598253428936, | |
| "step": 8, | |
| "step_time": 434.73393120799847 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.75, | |
| "completions/max_length": 1492.0, | |
| "completions/max_terminated_length": 1027.0, | |
| "completions/mean_length": 818.34375, | |
| "completions/mean_terminated_length": 474.0625, | |
| "completions/min_length": 192.0, | |
| "completions/min_terminated_length": 192.0, | |
| "entropy": 0.32139473129063845, | |
| "epoch": 0.072, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.006268487311899662, | |
| "kl": 0.0026504553206905257, | |
| "learning_rate": 4.8853481410001225e-06, | |
| "loss": -0.0089, | |
| "num_tokens": 456612.0, | |
| "reward": 0.19500000774860382, | |
| "reward_std": 0.29811495542526245, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.19500000774860382, | |
| "rewards/alfworld_rollout_reward_func/std": 0.46079298853874207, | |
| "rollout/action_tokens_mean": 818.34375, | |
| "rollout/episode_turns_mean": 26.109375, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.75, | |
| "sampling/importance_sampling_ratio/max": 2.6596250534057617, | |
| "sampling/importance_sampling_ratio/mean": 0.42990007996559143, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.3629698753356934, | |
| "sampling/sampling_logp_difference/mean": 0.019470183178782463, | |
| "step": 9, | |
| "step_time": 399.7356371579963 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.59375, | |
| "completions/max_length": 1381.0, | |
| "completions/max_terminated_length": 1015.0, | |
| "completions/mean_length": 710.296875, | |
| "completions/mean_terminated_length": 485.5384826660156, | |
| "completions/min_length": 193.0, | |
| "completions/min_terminated_length": 193.0, | |
| "entropy": 0.34738565888255835, | |
| "epoch": 0.08, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010129230096936226, | |
| "kl": 0.0015055066687637009, | |
| "learning_rate": 4.844380330367701e-06, | |
| "loss": 0.0042, | |
| "num_tokens": 502071.0, | |
| "reward": 0.3684375286102295, | |
| "reward_std": 0.3336261808872223, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.3684375286102295, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5042713284492493, | |
| "rollout/action_tokens_mean": 710.296875, | |
| "rollout/episode_turns_mean": 24.859375, | |
| "rollout/mismatch_mean": 0.015625, | |
| "rollout/mismatch_ratio": 0.015625, | |
| "rollout/truncated_ratio": 0.59375, | |
| "sampling/importance_sampling_ratio/max": 2.386782646179199, | |
| "sampling/importance_sampling_ratio/mean": 0.5169863104820251, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.2723636627197266, | |
| "sampling/sampling_logp_difference/mean": 0.022041790187358856, | |
| "step": 10, | |
| "step_time": 394.5470729360013 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.859375, | |
| "completions/max_length": 1352.0, | |
| "completions/max_terminated_length": 782.0, | |
| "completions/mean_length": 881.84375, | |
| "completions/mean_terminated_length": 523.4444580078125, | |
| "completions/min_length": 224.0, | |
| "completions/min_terminated_length": 224.0, | |
| "entropy": 0.3616171991452575, | |
| "epoch": 0.088, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.00803722906857729, | |
| "kl": 0.0017023698201228399, | |
| "learning_rate": 4.797394529050577e-06, | |
| "loss": 0.0111, | |
| "num_tokens": 558509.0, | |
| "reward": 0.08921875059604645, | |
| "reward_std": 0.2777005434036255, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.08921875059604645, | |
| "rewards/alfworld_rollout_reward_func/std": 0.36485835909843445, | |
| "rollout/action_tokens_mean": 881.84375, | |
| "rollout/episode_turns_mean": 28.40625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.859375, | |
| "sampling/importance_sampling_ratio/max": 2.7719504833221436, | |
| "sampling/importance_sampling_ratio/mean": 0.4752790629863739, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.038581132888794, | |
| "sampling/sampling_logp_difference/mean": 0.02142297849059105, | |
| "step": 11, | |
| "step_time": 396.1487058620023 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.671875, | |
| "completions/max_length": 2292.0, | |
| "completions/max_terminated_length": 901.0, | |
| "completions/mean_length": 860.28125, | |
| "completions/mean_terminated_length": 460.76190185546875, | |
| "completions/min_length": 116.0, | |
| "completions/min_terminated_length": 116.0, | |
| "entropy": 0.35516145126894116, | |
| "epoch": 0.096, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.011235510930418968, | |
| "kl": 0.0016268756971840048, | |
| "learning_rate": 4.744511348926855e-06, | |
| "loss": 0.0048, | |
| "num_tokens": 613567.0, | |
| "reward": 0.26890623569488525, | |
| "reward_std": 0.41516801714897156, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.26890626549720764, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4936746656894684, | |
| "rollout/action_tokens_mean": 860.28125, | |
| "rollout/episode_turns_mean": 25.484375, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.671875, | |
| "sampling/importance_sampling_ratio/max": 2.7596683502197266, | |
| "sampling/importance_sampling_ratio/mean": 0.5103189945220947, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.6376895904541016, | |
| "sampling/sampling_logp_difference/mean": 0.02156790718436241, | |
| "step": 12, | |
| "step_time": 421.574981908002 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.8125, | |
| "completions/max_length": 1382.0, | |
| "completions/max_terminated_length": 775.0, | |
| "completions/mean_length": 836.53125, | |
| "completions/mean_terminated_length": 586.4166870117188, | |
| "completions/min_length": 402.0, | |
| "completions/min_terminated_length": 402.0, | |
| "entropy": 0.35410969890654087, | |
| "epoch": 0.104, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010335148312151432, | |
| "kl": 0.0018077169770549517, | |
| "learning_rate": 4.685866540361456e-06, | |
| "loss": 0.0068, | |
| "num_tokens": 667105.0, | |
| "reward": 0.14281249046325684, | |
| "reward_std": 0.3198840916156769, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.14281249046325684, | |
| "rewards/alfworld_rollout_reward_func/std": 0.3984073996543884, | |
| "rollout/action_tokens_mean": 836.53125, | |
| "rollout/episode_turns_mean": 28.453125, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.8125, | |
| "sampling/importance_sampling_ratio/max": 2.6870784759521484, | |
| "sampling/importance_sampling_ratio/mean": 0.6008636951446533, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.14093017578125, | |
| "sampling/sampling_logp_difference/mean": 0.021617209538817406, | |
| "step": 13, | |
| "step_time": 369.507772967002 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6875, | |
| "completions/max_length": 1584.0, | |
| "completions/max_terminated_length": 1063.0, | |
| "completions/mean_length": 793.703125, | |
| "completions/mean_terminated_length": 525.4500122070312, | |
| "completions/min_length": 158.0, | |
| "completions/min_terminated_length": 158.0, | |
| "entropy": 0.34002805408090353, | |
| "epoch": 0.112, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.00711849657818675, | |
| "kl": 0.0019163258293701801, | |
| "learning_rate": 4.621610643736878e-06, | |
| "loss": 0.0062, | |
| "num_tokens": 717902.0, | |
| "reward": 0.2626562714576721, | |
| "reward_std": 0.3680095076560974, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.26265624165534973, | |
| "rewards/alfworld_rollout_reward_func/std": 0.489940881729126, | |
| "rollout/action_tokens_mean": 793.703125, | |
| "rollout/episode_turns_mean": 26.28125, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.6875, | |
| "sampling/importance_sampling_ratio/max": 2.815065860748291, | |
| "sampling/importance_sampling_ratio/mean": 0.45154738426208496, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.7529265880584717, | |
| "sampling/sampling_logp_difference/mean": 0.0205212514847517, | |
| "step": 14, | |
| "step_time": 327.6827599140015 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.84375, | |
| "completions/max_length": 1814.0, | |
| "completions/max_terminated_length": 890.0, | |
| "completions/mean_length": 909.109375, | |
| "completions/mean_terminated_length": 528.7999877929688, | |
| "completions/min_length": 242.0, | |
| "completions/min_terminated_length": 242.0, | |
| "entropy": 0.35556588135659695, | |
| "epoch": 0.12, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010462122038006783, | |
| "kl": 0.0016676104332873365, | |
| "learning_rate": 4.551908603018191e-06, | |
| "loss": -0.0076, | |
| "num_tokens": 776085.0, | |
| "reward": 0.10140626132488251, | |
| "reward_std": 0.36953580379486084, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.10140625387430191, | |
| "rewards/alfworld_rollout_reward_func/std": 0.38891327381134033, | |
| "rollout/action_tokens_mean": 909.109375, | |
| "rollout/episode_turns_mean": 28.0, | |
| "rollout/mismatch_mean": 0.015625, | |
| "rollout/mismatch_ratio": 0.015625, | |
| "rollout/truncated_ratio": 0.84375, | |
| "sampling/importance_sampling_ratio/max": 2.0437681674957275, | |
| "sampling/importance_sampling_ratio/mean": 0.3665631115436554, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 4.642125606536865, | |
| "sampling/sampling_logp_difference/mean": 0.021498428657650948, | |
| "step": 15, | |
| "step_time": 409.45475878100024 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.703125, | |
| "completions/max_length": 2069.0, | |
| "completions/max_terminated_length": 815.0, | |
| "completions/mean_length": 858.484375, | |
| "completions/mean_terminated_length": 363.0, | |
| "completions/min_length": 128.0, | |
| "completions/min_terminated_length": 128.0, | |
| "entropy": 0.3426384939812124, | |
| "epoch": 0.128, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.007739651948213577, | |
| "kl": 0.0015719706807431066, | |
| "learning_rate": 4.476939342344246e-06, | |
| "loss": -0.002, | |
| "num_tokens": 831028.0, | |
| "reward": 0.2515625059604645, | |
| "reward_std": 0.4047408103942871, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2515625059604645, | |
| "rewards/alfworld_rollout_reward_func/std": 0.48498478531837463, | |
| "rollout/action_tokens_mean": 858.484375, | |
| "rollout/episode_turns_mean": 25.015625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.703125, | |
| "sampling/importance_sampling_ratio/max": 2.197270631790161, | |
| "sampling/importance_sampling_ratio/mean": 0.3564271330833435, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.3733601570129395, | |
| "sampling/sampling_logp_difference/mean": 0.021193569526076317, | |
| "step": 16, | |
| "step_time": 341.3529842470016 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.765625, | |
| "completions/max_length": 1344.0, | |
| "completions/max_terminated_length": 872.0, | |
| "completions/mean_length": 800.890625, | |
| "completions/mean_terminated_length": 410.0000305175781, | |
| "completions/min_length": 199.0, | |
| "completions/min_terminated_length": 199.0, | |
| "entropy": 0.3335589300841093, | |
| "epoch": 0.136, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 1.4890336990356445, | |
| "kl": 0.8330444995917787, | |
| "learning_rate": 4.396895306731978e-06, | |
| "loss": 0.0005, | |
| "num_tokens": 882285.0, | |
| "reward": 0.18125000596046448, | |
| "reward_std": 0.24295468628406525, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.18124999105930328, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4482647478580475, | |
| "rollout/action_tokens_mean": 800.890625, | |
| "rollout/episode_turns_mean": 26.265625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.765625, | |
| "sampling/importance_sampling_ratio/max": 2.3350934982299805, | |
| "sampling/importance_sampling_ratio/mean": 0.37133079767227173, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 10.240796089172363, | |
| "sampling/sampling_logp_difference/mean": 0.020599324256181717, | |
| "step": 17, | |
| "step_time": 436.20165634099976 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.890625, | |
| "completions/max_length": 1359.0, | |
| "completions/max_terminated_length": 872.0, | |
| "completions/mean_length": 864.640625, | |
| "completions/mean_terminated_length": 638.7142944335938, | |
| "completions/min_length": 240.0, | |
| "completions/min_terminated_length": 240.0, | |
| "entropy": 0.3253911091014743, | |
| "epoch": 0.144, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.007197697646915913, | |
| "kl": 0.0016915635187615408, | |
| "learning_rate": 4.3119819680728e-06, | |
| "loss": -0.0099, | |
| "num_tokens": 937622.0, | |
| "reward": 0.06968750059604645, | |
| "reward_std": 0.19933855533599854, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.06968750059604645, | |
| "rewards/alfworld_rollout_reward_func/std": 0.3224163353443146, | |
| "rollout/action_tokens_mean": 864.640625, | |
| "rollout/episode_turns_mean": 28.9375, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.890625, | |
| "sampling/importance_sampling_ratio/max": 2.2717995643615723, | |
| "sampling/importance_sampling_ratio/mean": 0.5041627287864685, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.0254077911376953, | |
| "sampling/sampling_logp_difference/mean": 0.019858187064528465, | |
| "step": 18, | |
| "step_time": 437.1889309570033 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.703125, | |
| "completions/max_length": 2201.0, | |
| "completions/max_terminated_length": 1115.0, | |
| "completions/mean_length": 844.5, | |
| "completions/mean_terminated_length": 540.631591796875, | |
| "completions/min_length": 140.0, | |
| "completions/min_terminated_length": 140.0, | |
| "entropy": 0.33306339103728533, | |
| "epoch": 0.152, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010224551893770695, | |
| "kl": 0.001574573227117071, | |
| "learning_rate": 4.222417297689217e-06, | |
| "loss": 0.0213, | |
| "num_tokens": 991670.0, | |
| "reward": 0.24437499046325684, | |
| "reward_std": 0.42425790429115295, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.24437500536441803, | |
| "rewards/alfworld_rollout_reward_func/std": 0.48038968443870544, | |
| "rollout/action_tokens_mean": 844.5, | |
| "rollout/episode_turns_mean": 26.171875, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.703125, | |
| "sampling/importance_sampling_ratio/max": 2.5866730213165283, | |
| "sampling/importance_sampling_ratio/mean": 0.5376668572425842, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.632045865058899, | |
| "sampling/sampling_logp_difference/mean": 0.01941317319869995, | |
| "step": 19, | |
| "step_time": 348.9997736020032 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.75, | |
| "completions/max_length": 1764.0, | |
| "completions/max_terminated_length": 1085.0, | |
| "completions/mean_length": 937.640625, | |
| "completions/mean_terminated_length": 530.6875, | |
| "completions/min_length": 201.0, | |
| "completions/min_terminated_length": 201.0, | |
| "entropy": 0.3477782104164362, | |
| "epoch": 0.16, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010612000711262226, | |
| "kl": 0.001602184345756541, | |
| "learning_rate": 4.128431206805556e-06, | |
| "loss": 0.0044, | |
| "num_tokens": 1051679.0, | |
| "reward": 0.1901562511920929, | |
| "reward_std": 0.39348262548446655, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.1901562511920929, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4530303478240967, | |
| "rollout/action_tokens_mean": 937.640625, | |
| "rollout/episode_turns_mean": 26.6875, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.75, | |
| "sampling/importance_sampling_ratio/max": 2.5002288818359375, | |
| "sampling/importance_sampling_ratio/mean": 0.4413934350013733, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.34731388092041, | |
| "sampling/sampling_logp_difference/mean": 0.0198683924973011, | |
| "step": 20, | |
| "step_time": 353.7044870629943 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.65625, | |
| "completions/max_length": 1475.0, | |
| "completions/max_terminated_length": 939.0, | |
| "completions/mean_length": 800.453125, | |
| "completions/mean_terminated_length": 418.0909118652344, | |
| "completions/min_length": 99.0, | |
| "completions/min_terminated_length": 99.0, | |
| "entropy": 0.36278699059039354, | |
| "epoch": 0.168, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010538731701672077, | |
| "kl": 0.001873640143458033, | |
| "learning_rate": 4.030264956369158e-06, | |
| "loss": -0.0018, | |
| "num_tokens": 1102908.0, | |
| "reward": 0.2993749976158142, | |
| "reward_std": 0.40504124760627747, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2993749976158142, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4982314109802246, | |
| "rollout/action_tokens_mean": 800.453125, | |
| "rollout/episode_turns_mean": 24.4375, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.65625, | |
| "sampling/importance_sampling_ratio/max": 2.447692632675171, | |
| "sampling/importance_sampling_ratio/mean": 0.448123574256897, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.2475838661193848, | |
| "sampling/sampling_logp_difference/mean": 0.022293493151664734, | |
| "step": 21, | |
| "step_time": 452.2219315199982 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.625, | |
| "completions/max_length": 1566.0, | |
| "completions/max_terminated_length": 929.0, | |
| "completions/mean_length": 743.703125, | |
| "completions/mean_terminated_length": 460.91668701171875, | |
| "completions/min_length": 137.0, | |
| "completions/min_terminated_length": 137.0, | |
| "entropy": 0.3329707160592079, | |
| "epoch": 0.176, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.007643942255526781, | |
| "kl": 0.0019213874111301266, | |
| "learning_rate": 3.9281705377369814e-06, | |
| "loss": 0.0059, | |
| "num_tokens": 1150505.0, | |
| "reward": 0.32625001668930054, | |
| "reward_std": 0.4726526141166687, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.32624998688697815, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5121089816093445, | |
| "rollout/action_tokens_mean": 743.703125, | |
| "rollout/episode_turns_mean": 24.796875, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.625, | |
| "sampling/importance_sampling_ratio/max": 2.206357479095459, | |
| "sampling/importance_sampling_ratio/mean": 0.4393410086631775, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.366598129272461, | |
| "sampling/sampling_logp_difference/mean": 0.020861441269516945, | |
| "step": 22, | |
| "step_time": 350.210106904995 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.65625, | |
| "completions/max_length": 1494.0, | |
| "completions/max_terminated_length": 1159.0, | |
| "completions/mean_length": 797.828125, | |
| "completions/mean_terminated_length": 504.04547119140625, | |
| "completions/min_length": 87.0, | |
| "completions/min_terminated_length": 87.0, | |
| "entropy": 0.35823060059919953, | |
| "epoch": 0.184, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010937543585896492, | |
| "kl": 0.00202080328745069, | |
| "learning_rate": 3.8224100258174066e-06, | |
| "loss": 0.0064, | |
| "num_tokens": 1201566.0, | |
| "reward": 0.3057812452316284, | |
| "reward_std": 0.36121153831481934, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.3057812452316284, | |
| "rewards/alfworld_rollout_reward_func/std": 0.49343568086624146, | |
| "rollout/action_tokens_mean": 797.828125, | |
| "rollout/episode_turns_mean": 25.328125, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.65625, | |
| "sampling/importance_sampling_ratio/max": 2.8112552165985107, | |
| "sampling/importance_sampling_ratio/mean": 0.43328696489334106, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.2014861106872559, | |
| "sampling/sampling_logp_difference/mean": 0.022007007151842117, | |
| "step": 23, | |
| "step_time": 299.4183257309978 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.75, | |
| "completions/max_length": 1409.0, | |
| "completions/max_terminated_length": 962.0, | |
| "completions/mean_length": 749.84375, | |
| "completions/mean_terminated_length": 406.0, | |
| "completions/min_length": 135.0, | |
| "completions/min_terminated_length": 135.0, | |
| "entropy": 0.314591265283525, | |
| "epoch": 0.192, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.008087356574833393, | |
| "kl": 0.0017565299822308589, | |
| "learning_rate": 3.7132549063277033e-06, | |
| "loss": -0.0022, | |
| "num_tokens": 1249556.0, | |
| "reward": 0.20937500894069672, | |
| "reward_std": 0.2735254764556885, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.20937500894069672, | |
| "rewards/alfworld_rollout_reward_func/std": 0.45049840211868286, | |
| "rollout/action_tokens_mean": 749.84375, | |
| "rollout/episode_turns_mean": 25.53125, | |
| "rollout/mismatch_mean": 0.03125, | |
| "rollout/mismatch_ratio": 0.03125, | |
| "rollout/truncated_ratio": 0.75, | |
| "sampling/importance_sampling_ratio/max": 2.822704792022705, | |
| "sampling/importance_sampling_ratio/mean": 0.4837469756603241, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.2067627906799316, | |
| "sampling/sampling_logp_difference/mean": 0.021238919347524643, | |
| "step": 24, | |
| "step_time": 423.74345705999986 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.828125, | |
| "completions/max_length": 1469.0, | |
| "completions/max_terminated_length": 854.0, | |
| "completions/mean_length": 757.296875, | |
| "completions/mean_terminated_length": 445.2727355957031, | |
| "completions/min_length": 211.0, | |
| "completions/min_terminated_length": 211.0, | |
| "entropy": 0.3146928045898676, | |
| "epoch": 0.2, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.005950183607637882, | |
| "kl": 0.0024588965916336747, | |
| "learning_rate": 3.600985378894086e-06, | |
| "loss": -0.0029, | |
| "num_tokens": 1298023.0, | |
| "reward": 0.12843748927116394, | |
| "reward_std": 0.3103031814098358, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.12843750417232513, | |
| "rewards/alfworld_rollout_reward_func/std": 0.39419689774513245, | |
| "rollout/action_tokens_mean": 757.296875, | |
| "rollout/episode_turns_mean": 27.515625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.828125, | |
| "sampling/importance_sampling_ratio/max": 1.9042965173721313, | |
| "sampling/importance_sampling_ratio/mean": 0.30797556042671204, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.0986688137054443, | |
| "sampling/sampling_logp_difference/mean": 0.0199856199324131, | |
| "step": 25, | |
| "step_time": 379.6568018289963 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.796875, | |
| "completions/max_length": 1607.0, | |
| "completions/max_terminated_length": 975.0, | |
| "completions/mean_length": 820.5, | |
| "completions/mean_terminated_length": 396.923095703125, | |
| "completions/min_length": 174.0, | |
| "completions/min_terminated_length": 174.0, | |
| "entropy": 0.301429130602628, | |
| "epoch": 0.208, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.006396389566361904, | |
| "kl": 0.001548040645502624, | |
| "learning_rate": 3.4858896377832966e-06, | |
| "loss": 0.0009, | |
| "num_tokens": 1350535.0, | |
| "reward": 0.1639062464237213, | |
| "reward_std": 0.32607388496398926, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.1639062464237213, | |
| "rewards/alfworld_rollout_reward_func/std": 0.418832927942276, | |
| "rollout/action_tokens_mean": 820.5, | |
| "rollout/episode_turns_mean": 26.59375, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.796875, | |
| "sampling/importance_sampling_ratio/max": 2.7253878116607666, | |
| "sampling/importance_sampling_ratio/mean": 0.46614933013916016, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.6588082313537598, | |
| "sampling/sampling_logp_difference/mean": 0.018608851358294487, | |
| "step": 26, | |
| "step_time": 458.5120370709956 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.8125, | |
| "completions/max_length": 1363.0, | |
| "completions/max_terminated_length": 851.0, | |
| "completions/mean_length": 843.46875, | |
| "completions/mean_terminated_length": 518.0833740234375, | |
| "completions/min_length": 123.0, | |
| "completions/min_terminated_length": 123.0, | |
| "entropy": 0.3490098095498979, | |
| "epoch": 0.216, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.00987387914210558, | |
| "kl": 0.001589072278875392, | |
| "learning_rate": 3.3682631321120507e-06, | |
| "loss": -0.0109, | |
| "num_tokens": 1404517.0, | |
| "reward": 0.14921875298023224, | |
| "reward_std": 0.3126556873321533, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.14921875298023224, | |
| "rewards/alfworld_rollout_reward_func/std": 0.40770089626312256, | |
| "rollout/action_tokens_mean": 843.46875, | |
| "rollout/episode_turns_mean": 27.703125, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.8125, | |
| "sampling/importance_sampling_ratio/max": 2.323033332824707, | |
| "sampling/importance_sampling_ratio/mean": 0.4162726402282715, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.8558279275894165, | |
| "sampling/sampling_logp_difference/mean": 0.020594514906406403, | |
| "step": 27, | |
| "step_time": 366.20077905700964 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5625, | |
| "completions/max_length": 2369.0, | |
| "completions/max_terminated_length": 1151.0, | |
| "completions/mean_length": 727.859375, | |
| "completions/mean_terminated_length": 404.3571472167969, | |
| "completions/min_length": 97.0, | |
| "completions/min_terminated_length": 97.0, | |
| "entropy": 0.30788369150832295, | |
| "epoch": 0.224, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.01545465737581253, | |
| "kl": 0.0016130298517964548, | |
| "learning_rate": 3.248407807433396e-06, | |
| "loss": 0.012, | |
| "num_tokens": 1451100.0, | |
| "reward": 0.3981249928474426, | |
| "reward_std": 0.5213440656661987, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.3981249928474426, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5216788649559021, | |
| "rollout/action_tokens_mean": 727.859375, | |
| "rollout/episode_turns_mean": 22.984375, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.5625, | |
| "sampling/importance_sampling_ratio/max": 2.7356457710266113, | |
| "sampling/importance_sampling_ratio/mean": 0.6839978694915771, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.65842604637146, | |
| "sampling/sampling_logp_difference/mean": 0.019816581159830093, | |
| "step": 28, | |
| "step_time": 317.7198553449962 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.75, | |
| "completions/max_length": 1781.0, | |
| "completions/max_terminated_length": 980.0, | |
| "completions/mean_length": 844.03125, | |
| "completions/mean_terminated_length": 524.75, | |
| "completions/min_length": 153.0, | |
| "completions/min_terminated_length": 153.0, | |
| "entropy": 0.3514281096868217, | |
| "epoch": 0.232, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010329142212867737, | |
| "kl": 0.0034811663499567658, | |
| "learning_rate": 3.1266313306468018e-06, | |
| "loss": -0.0153, | |
| "num_tokens": 1505118.0, | |
| "reward": 0.20656250417232513, | |
| "reward_std": 0.29942071437835693, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.20656250417232513, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4480122923851013, | |
| "rollout/action_tokens_mean": 844.03125, | |
| "rollout/episode_turns_mean": 26.890625, | |
| "rollout/mismatch_mean": 0.015625, | |
| "rollout/mismatch_ratio": 0.015625, | |
| "rollout/truncated_ratio": 0.75, | |
| "sampling/importance_sampling_ratio/max": 2.6475582122802734, | |
| "sampling/importance_sampling_ratio/mean": 0.39373910427093506, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 5.319100856781006, | |
| "sampling/sampling_logp_difference/mean": 0.022610869258642197, | |
| "step": 29, | |
| "step_time": 372.01922321400525 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.75, | |
| "completions/max_length": 1465.0, | |
| "completions/max_terminated_length": 804.0, | |
| "completions/mean_length": 781.265625, | |
| "completions/mean_terminated_length": 451.8125, | |
| "completions/min_length": 172.0, | |
| "completions/min_terminated_length": 172.0, | |
| "entropy": 0.2927012937143445, | |
| "epoch": 0.24, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.006770053878426552, | |
| "kl": 0.0015057786895340541, | |
| "learning_rate": 3.0032463002216504e-06, | |
| "loss": 0.0038, | |
| "num_tokens": 1555119.0, | |
| "reward": 0.21671874821186066, | |
| "reward_std": 0.24240338802337646, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.21671876311302185, | |
| "rewards/alfworld_rollout_reward_func/std": 0.44675213098526, | |
| "rollout/action_tokens_mean": 781.265625, | |
| "rollout/episode_turns_mean": 26.21875, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.75, | |
| "sampling/importance_sampling_ratio/max": 2.8479506969451904, | |
| "sampling/importance_sampling_ratio/mean": 0.49506521224975586, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.7825241088867188, | |
| "sampling/sampling_logp_difference/mean": 0.01828092336654663, | |
| "step": 30, | |
| "step_time": 382.84014304800075 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.8125, | |
| "completions/max_length": 1591.0, | |
| "completions/max_terminated_length": 899.0, | |
| "completions/mean_length": 807.0625, | |
| "completions/mean_terminated_length": 503.8333435058594, | |
| "completions/min_length": 153.0, | |
| "completions/min_terminated_length": 153.0, | |
| "entropy": 0.33275228925049305, | |
| "epoch": 0.248, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.009322401136159897, | |
| "kl": 0.0018803979564836482, | |
| "learning_rate": 2.878569443761442e-06, | |
| "loss": -0.0078, | |
| "num_tokens": 1606771.0, | |
| "reward": 0.14656250178813934, | |
| "reward_std": 0.337169349193573, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.14656248688697815, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4100453853607178, | |
| "rollout/action_tokens_mean": 807.0625, | |
| "rollout/episode_turns_mean": 27.546875, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.8125, | |
| "sampling/importance_sampling_ratio/max": 2.9021573066711426, | |
| "sampling/importance_sampling_ratio/mean": 0.42428821325302124, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.855207920074463, | |
| "sampling/sampling_logp_difference/mean": 0.021321609616279602, | |
| "step": 31, | |
| "step_time": 494.6054373920015 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.796875, | |
| "completions/max_length": 1383.0, | |
| "completions/max_terminated_length": 1038.0, | |
| "completions/mean_length": 791.734375, | |
| "completions/mean_terminated_length": 412.923095703125, | |
| "completions/min_length": 125.0, | |
| "completions/min_terminated_length": 125.0, | |
| "entropy": 0.38143961410969496, | |
| "epoch": 0.256, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010985196568071842, | |
| "kl": 0.0017831376208050642, | |
| "learning_rate": 2.752920804968581e-06, | |
| "loss": -0.0093, | |
| "num_tokens": 1657442.0, | |
| "reward": 0.14656248688697815, | |
| "reward_std": 0.27167972922325134, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.14656250178813934, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4263809323310852, | |
| "rollout/action_tokens_mean": 791.734375, | |
| "rollout/episode_turns_mean": 26.5, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.796875, | |
| "sampling/importance_sampling_ratio/max": 2.170595169067383, | |
| "sampling/importance_sampling_ratio/mean": 0.39966434240341187, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.398648977279663, | |
| "sampling/sampling_logp_difference/mean": 0.022872522473335266, | |
| "step": 32, | |
| "step_time": 305.8924537160019 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.734375, | |
| "completions/max_length": 1564.0, | |
| "completions/max_terminated_length": 1151.0, | |
| "completions/mean_length": 914.953125, | |
| "completions/mean_terminated_length": 616.4705810546875, | |
| "completions/min_length": 291.0, | |
| "completions/min_terminated_length": 291.0, | |
| "entropy": 0.39743291586637497, | |
| "epoch": 0.264, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.010168999433517456, | |
| "kl": 0.0017962153833650518, | |
| "learning_rate": 2.626622922096782e-06, | |
| "loss": -0.0045, | |
| "num_tokens": 1715999.0, | |
| "reward": 0.21843750774860382, | |
| "reward_std": 0.4097728729248047, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.21843750774860382, | |
| "rewards/alfworld_rollout_reward_func/std": 0.46011462807655334, | |
| "rollout/action_tokens_mean": 914.953125, | |
| "rollout/episode_turns_mean": 27.140625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.734375, | |
| "sampling/importance_sampling_ratio/max": 2.6107707023620605, | |
| "sampling/importance_sampling_ratio/mean": 0.386021226644516, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.2772090435028076, | |
| "sampling/sampling_logp_difference/mean": 0.02191011980175972, | |
| "step": 33, | |
| "step_time": 360.4347860389935 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.796875, | |
| "completions/max_length": 1527.0, | |
| "completions/max_terminated_length": 743.0, | |
| "completions/mean_length": 862.9375, | |
| "completions/mean_terminated_length": 506.3077087402344, | |
| "completions/min_length": 254.0, | |
| "completions/min_terminated_length": 254.0, | |
| "entropy": 0.35848523303866386, | |
| "epoch": 0.272, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.01134358998388052, | |
| "kl": 0.001760420509526739, | |
| "learning_rate": 2.5e-06, | |
| "loss": 0.016, | |
| "num_tokens": 1771227.0, | |
| "reward": 0.15781250596046448, | |
| "reward_std": 0.29862138628959656, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.15781249105930328, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4202209413051605, | |
| "rollout/action_tokens_mean": 862.9375, | |
| "rollout/episode_turns_mean": 27.515625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.796875, | |
| "sampling/importance_sampling_ratio/max": 2.3946166038513184, | |
| "sampling/importance_sampling_ratio/mean": 0.41097065806388855, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.3659703731536865, | |
| "sampling/sampling_logp_difference/mean": 0.020839743316173553, | |
| "step": 34, | |
| "step_time": 255.86244797499785 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.640625, | |
| "completions/max_length": 1503.0, | |
| "completions/max_terminated_length": 1127.0, | |
| "completions/mean_length": 801.078125, | |
| "completions/mean_terminated_length": 446.0434875488281, | |
| "completions/min_length": 121.0, | |
| "completions/min_terminated_length": 121.0, | |
| "entropy": 0.33974334690719843, | |
| "epoch": 0.28, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.007748698350042105, | |
| "kl": 0.0017098757416533772, | |
| "learning_rate": 2.3733770779032185e-06, | |
| "loss": 0.0012, | |
| "num_tokens": 1822496.0, | |
| "reward": 0.3109375238418579, | |
| "reward_std": 0.34154438972473145, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.3109375238418579, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5012863874435425, | |
| "rollout/action_tokens_mean": 801.078125, | |
| "rollout/episode_turns_mean": 24.203125, | |
| "rollout/mismatch_mean": 0.015625, | |
| "rollout/mismatch_ratio": 0.015625, | |
| "rollout/truncated_ratio": 0.640625, | |
| "sampling/importance_sampling_ratio/max": 2.6798601150512695, | |
| "sampling/importance_sampling_ratio/mean": 0.4648974537849426, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.4488577842712402, | |
| "sampling/sampling_logp_difference/mean": 0.020977256819605827, | |
| "step": 35, | |
| "step_time": 369.7442883429976 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5625, | |
| "completions/max_length": 1817.0, | |
| "completions/max_terminated_length": 1098.0, | |
| "completions/mean_length": 763.078125, | |
| "completions/mean_terminated_length": 383.21429443359375, | |
| "completions/min_length": 90.0, | |
| "completions/min_terminated_length": 90.0, | |
| "entropy": 0.37786182574927807, | |
| "epoch": 0.288, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.012169571593403816, | |
| "kl": 0.0024242461659014225, | |
| "learning_rate": 2.24707919503142e-06, | |
| "loss": 0.0067, | |
| "num_tokens": 1871333.0, | |
| "reward": 0.3814062476158142, | |
| "reward_std": 0.325703501701355, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.3814062476158142, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5243707299232483, | |
| "rollout/action_tokens_mean": 763.078125, | |
| "rollout/episode_turns_mean": 22.328125, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.5625, | |
| "sampling/importance_sampling_ratio/max": 2.630308151245117, | |
| "sampling/importance_sampling_ratio/mean": 0.4907624125480652, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.9331345558166504, | |
| "sampling/sampling_logp_difference/mean": 0.022463176399469376, | |
| "step": 36, | |
| "step_time": 452.8674500970046 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.453125, | |
| "completions/max_length": 1560.0, | |
| "completions/max_terminated_length": 1173.0, | |
| "completions/mean_length": 592.9375, | |
| "completions/mean_terminated_length": 322.4285583496094, | |
| "completions/min_length": 102.0, | |
| "completions/min_terminated_length": 102.0, | |
| "entropy": 0.2960165496915579, | |
| "epoch": 0.296, | |
| "frac_reward_zero_std": 0.125, | |
| "grad_norm": 0.009033096954226494, | |
| "kl": 0.0017318885384156602, | |
| "learning_rate": 2.1214305562385592e-06, | |
| "loss": 0.0028, | |
| "num_tokens": 1909281.0, | |
| "reward": 0.5143750309944153, | |
| "reward_std": 0.3860568404197693, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.5143750309944153, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5290928483009338, | |
| "rollout/action_tokens_mean": 592.9375, | |
| "rollout/episode_turns_mean": 19.015625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.453125, | |
| "sampling/importance_sampling_ratio/max": 2.8567276000976562, | |
| "sampling/importance_sampling_ratio/mean": 0.5531047582626343, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.304647445678711, | |
| "sampling/sampling_logp_difference/mean": 0.01983683556318283, | |
| "step": 37, | |
| "step_time": 388.3879080370025 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.53125, | |
| "completions/max_length": 1343.0, | |
| "completions/max_terminated_length": 792.0, | |
| "completions/mean_length": 662.140625, | |
| "completions/mean_terminated_length": 343.8333435058594, | |
| "completions/min_length": 77.0, | |
| "completions/min_terminated_length": 77.0, | |
| "entropy": 0.32609579246491194, | |
| "epoch": 0.304, | |
| "frac_reward_zero_std": 0.125, | |
| "grad_norm": 0.014773277565836906, | |
| "kl": 0.0018760439170364407, | |
| "learning_rate": 1.9967536997783495e-06, | |
| "loss": 0.0014, | |
| "num_tokens": 1951658.0, | |
| "reward": 0.4350000023841858, | |
| "reward_std": 0.3387959599494934, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.4350000023841858, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5278016924858093, | |
| "rollout/action_tokens_mean": 662.140625, | |
| "rollout/episode_turns_mean": 21.40625, | |
| "rollout/mismatch_mean": 0.015625, | |
| "rollout/mismatch_ratio": 0.015625, | |
| "rollout/truncated_ratio": 0.53125, | |
| "sampling/importance_sampling_ratio/max": 2.9260754585266113, | |
| "sampling/importance_sampling_ratio/mean": 0.563366711139679, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.768939971923828, | |
| "sampling/sampling_logp_difference/mean": 0.022423237562179565, | |
| "step": 38, | |
| "step_time": 436.1079989810023 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.734375, | |
| "completions/max_length": 1869.0, | |
| "completions/max_terminated_length": 1018.0, | |
| "completions/mean_length": 943.40625, | |
| "completions/mean_terminated_length": 584.4117431640625, | |
| "completions/min_length": 237.0, | |
| "completions/min_terminated_length": 237.0, | |
| "entropy": 0.3758079996332526, | |
| "epoch": 0.312, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.01313227228820324, | |
| "kl": 0.0022639515991613735, | |
| "learning_rate": 1.8733686693531986e-06, | |
| "loss": -0.0025, | |
| "num_tokens": 2012036.0, | |
| "reward": 0.21578124165534973, | |
| "reward_std": 0.4323212504386902, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.21578124165534973, | |
| "rewards/alfworld_rollout_reward_func/std": 0.46428921818733215, | |
| "rollout/action_tokens_mean": 943.40625, | |
| "rollout/episode_turns_mean": 26.90625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.734375, | |
| "sampling/importance_sampling_ratio/max": 2.5490753650665283, | |
| "sampling/importance_sampling_ratio/mean": 0.3670840859413147, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.389880657196045, | |
| "sampling/sampling_logp_difference/mean": 0.021743088960647583, | |
| "step": 39, | |
| "step_time": 354.37862886100265 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.671875, | |
| "completions/max_length": 1334.0, | |
| "completions/max_terminated_length": 1062.0, | |
| "completions/mean_length": 701.15625, | |
| "completions/mean_terminated_length": 308.1428527832031, | |
| "completions/min_length": 155.0, | |
| "completions/min_terminated_length": 155.0, | |
| "entropy": 0.2994346688501537, | |
| "epoch": 0.32, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.009722444228827953, | |
| "kl": 0.0016241020402958384, | |
| "learning_rate": 1.7515921925666053e-06, | |
| "loss": -0.0206, | |
| "num_tokens": 2056910.0, | |
| "reward": 0.29640626907348633, | |
| "reward_std": 0.23870769143104553, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.29640626907348633, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4918813109397888, | |
| "rollout/action_tokens_mean": 701.15625, | |
| "rollout/episode_turns_mean": 23.625, | |
| "rollout/mismatch_mean": 0.0, | |
| "rollout/mismatch_ratio": 0.0, | |
| "rollout/truncated_ratio": 0.671875, | |
| "sampling/importance_sampling_ratio/max": 2.825843572616577, | |
| "sampling/importance_sampling_ratio/mean": 0.5806776881217957, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.3793482780456543, | |
| "sampling/sampling_logp_difference/mean": 0.01931983232498169, | |
| "step": 40, | |
| "step_time": 355.1650720970065 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 64, | |
| "num_input_tokens_seen": 2056910, | |
| "num_train_epochs": 1, | |
| "save_steps": 20, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |